Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Add semantic search health checks (#78299)

Chris Truter1ba3a7c05048
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index e6d1d7c1a8c..08c75d27b46 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -3597,6 +3597,7 @@
config
connection-pool
events
+ health-inspector
lib-be
llm
metabot
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj
index 8f6fa7028be..5b41441e212 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj
@@ -3,6 +3,7 @@
(:require
[clojure.string :as str]
[medley.core :as m]
+ [metabase-enterprise.semantic-search.dlq :as semantic.dlq]
[metabase-enterprise.semantic-search.embedders]
[metabase-enterprise.semantic-search.embedding :as semantic.embedding]
[metabase-enterprise.semantic-search.env :as semantic.env]
@@ -17,6 +18,8 @@
[metabase.search.engine :as search.engine]
[metabase.tracing.core :as tracing]
[metabase.util.log :as log]
+ [next.jdbc :as jdbc]
+ [next.jdbc.result-set :as jdbc.rs]
[potemkin :as p]
[toucan2.realize :as t2.realize]))
@@ -35,6 +38,10 @@
(defn- index-active? [pgvector index-metadata]
(boolean (semantic.index-metadata/get-active-index-state pgvector index-metadata)))
+(defn- capture-repair-snapshot-at [pgvector]
+ (:snapshot_at (jdbc/execute-one! pgvector ["SELECT clock_timestamp() AS snapshot_at"]
+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})))
+
(defenterprise supported?
"Enterprise implementation of semantic search engine support check."
:feature :semantic-search
@@ -184,17 +191,33 @@
(defenterprise repair-index!
"Brings the semantic search index into consistency with the provided document set.
- Does not fully reinitialize the index, but will add missing documents and remove stale ones."
+ Does not fully reinitialize the index, but will add missing documents and remove stale ones.
+ Returns the repaired index ID, its stale-orphan count, and the pgvector timestamp captured before reading
+ the canonical stream."
:feature :semantic-search
[searchable-documents]
(let [pgvector (semantic.env/get-pgvector-datasource!)
- index-metadata (semantic.env/get-index-metadata)]
- (if-not (index-active? pgvector index-metadata)
+ index-metadata (semantic.env/get-index-metadata)
+ snapshot-at (capture-repair-snapshot-at pgvector)
+ active-state (semantic.index-metadata/get-active-index-state pgvector index-metadata)]
+ (if-not active-state
;; Semantic can become active at runtime (license applied, or added to additional-search-engines)
;; without init! ever having run; initializing here lets the periodic repair task backfill the index.
(do
(log/info "No active semantic index, initializing it instead of repairing")
- (init! searchable-documents {}))
+ (semantic.pgvector-api/init-semantic-search!
+ pgvector index-metadata (semantic.env/get-configured-embedding-model) {})
+ (semantic.repair/with-repair-table!
+ pgvector
+ index-metadata
+ (fn [repair-table-name]
+ (semantic.pgvector-api/gate-updates! pgvector index-metadata searchable-documents
+ :repair-table repair-table-name)
+ {:index-id (-> (semantic.index-metadata/get-active-index-state pgvector index-metadata)
+ :metadata-row
+ :id)
+ :orphans 0
+ :snapshot-at snapshot-at})))
(semantic.repair/with-repair-table!
pgvector
index-metadata
@@ -202,11 +225,27 @@
;; Re-gate all provided documents, populating the repair table as we go
(semantic.pgvector-api/gate-updates! pgvector index-metadata searchable-documents
:repair-table repair-table-name)
- ;; Find documents in the gate table that are not in the provided searchable-documents, and gate deletes for them
- (when-let [ids-by-model (semantic.repair/find-lost-deletes-by-model pgvector (:gate-table-name index-metadata) repair-table-name)]
- (doseq [[model ids] ids-by-model]
- (log/infof "Repairing lost deletes for model %s: deleting %d documents" model (count ids))
- (semantic.pgvector-api/gate-deletes! pgvector index-metadata model ids))))))))
+ ;; Counted BEFORE this run's gate-deletes, so the lost deletes found below (in-flight cleanup the
+ ;; indexer typically clears within minutes) don't read as a garbage spike that stands until the
+ ;; next hourly repair; see [[semantic.repair/count-stale-orphans]] for the full contract.
+ (let [orphans (semantic.repair/count-stale-orphans pgvector
+ (-> active-state :index :table-name)
+ (:gate-table-name index-metadata)
+ repair-table-name
+ (semantic.dlq/dlq-table-name-kw
+ index-metadata
+ (-> active-state :metadata-row :id))
+ (:metadata-row active-state))]
+ ;; Gate deletes for documents absent from the current searchable set.
+ (when-let [ids-by-model
+ (semantic.repair/find-lost-deletes-by-model
+ pgvector (:gate-table-name index-metadata) repair-table-name)]
+ (doseq [[model ids] ids-by-model]
+ (log/infof "Repairing lost deletes for model %s: deleting %d documents" model (count ids))
+ (semantic.pgvector-api/gate-deletes! pgvector index-metadata model ids)))
+ {:index-id (-> active-state :metadata-row :id)
+ :orphans orphans
+ :snapshot-at snapshot-at}))))))
(comment
(update-index! [{:model "card"
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration.clj
index 15e3772f6d5..ce43fba805a 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration.clj
@@ -88,7 +88,11 @@
:else
(log/infof "Database schema version (%d) is newer than code version (%d). Not performing migration."
- db-version semantic.db.migration.impl/schema-version)))
+ db-version semantic.db.migration.impl/schema-version))
+ ;; Existing schema-version-2 databases predate these additive columns. This compatibility pass must also
+ ;; run when the migration version already matches; newer schemas remain untouched by older code.
+ (when (<= db-version semantic.db.migration.impl/schema-version)
+ (semantic.db.migration.impl/ensure-schema-compatibility! tx opts)))
nil)
(defn- index-metadata-table-exists?
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration/impl.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration/impl.clj
index 27e0ee9f45b..cbbbd357d2a 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration/impl.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration/impl.clj
@@ -66,6 +66,11 @@
(semantic.index-metadata/create-tables-if-not-exists! tx index-metadata)
(semantic.index-metadata/ensure-control-row-exists! tx index-metadata))
+(defn ensure-schema-compatibility!
+ "Apply non-destructive additions that existing databases at the current schema version also need."
+ [tx {:keys [index-metadata]}]
+ (semantic.index-metadata/ensure-health-metric-columns! tx index-metadata))
+
(def dynamic-schema-version
"Code version of dynamic schema (the index_<provider>_<model>_<dims> index tables). If higher than what's found in
db dynamic schema migration will be attempted."
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/health.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/health.clj
new file mode 100644
index 00000000000..04ac4c6d72b
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/health.clj
@@ -0,0 +1,298 @@
+(ns metabase-enterprise.semantic-search.health
+ "Health-inspector checks and index metrics for semantic search.
+
+ Semantic search degrades silently -- a missing index, stalled indexer, or unreachable embedder just falls
+ back to appdb search, unnoticed. This check surfaces that.
+
+ :health can take the following values:
+ - nil = not enabled (check omitted)
+ - 0 = enabled but degraded (`:message` names the cause)
+ - 100 = serving"
+ (:require
+ [clojure.core.memoize :as memoize]
+ [clojure.string :as str]
+ [metabase-enterprise.semantic-search.dlq :as semantic.dlq]
+ [metabase-enterprise.semantic-search.embedding :as semantic.embedding]
+ [metabase-enterprise.semantic-search.embedding-health :as embedding-health]
+ [metabase-enterprise.semantic-search.env :as semantic.env]
+ [metabase-enterprise.semantic-search.index :as semantic.index]
+ [metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata]
+ [metabase-enterprise.semantic-search.settings :as semantic.settings]
+ [metabase-enterprise.semantic-search.util :as semantic.util]
+ [metabase.health-inspector.core :as health-inspector]
+ [metabase.search.config :as search.config]
+ [metabase.search.index-health :as search.index-health]
+ [metabase.util.log :as log]
+ [next.jdbc :as jdbc]
+ [next.jdbc.result-set :as jdbc.rs]))
+
+(set! *warn-on-reflection* true)
+
+(defn- active-index-queryable?
+ "Whether a trivial `SELECT ... LIMIT 1` against the active index table succeeds -- i.e. pgvector is
+ reachable and the table exists and is queryable."
+ [pgvector table-name]
+ (try
+ (jdbc/execute-one! pgvector [(format "SELECT 1 FROM %s LIMIT 1" (semantic.util/quote-table table-name))])
+ true
+ (catch InterruptedException e
+ (throw e))
+ (catch Exception _ false)))
+
+(defn- required-hnsw-index-present?
+ "Whether the active index satisfies the configured default vector strategy."
+ [pgvector index]
+ (try
+ (or (not (contains? search.config/hnsw-index-backed-strategies
+ (semantic.settings/semantic-search-vector-strategy)))
+ (semantic.util/index-ready?
+ pgvector
+ (semantic.index/schema-qualified-index-name index (semantic.index/hnsw-index-name index))))
+ (catch InterruptedException e
+ (throw e))
+ (catch Exception _ false)))
+
+(defn index-health-check
+ "Health-inspector check for semantic search, registered as `:semantic-search-index`.
+ Returns nil when the semantic engine isn't active on this instance (so the check is omitted), otherwise a
+ `{:health :message}` map: healthy when the active index matches the configured model and is queryable,
+ un-stalled, strategy-ready, and backed by a reachable embedder; degraded otherwise."
+ []
+ ;; Active, not merely available: an available-but-inactive engine (another engine selected) has no index
+ ;; by design, and must not read as a standing "No active semantic search index" incident. Active folds in
+ ;; the license and kill switch, so a disabled instance neither records runs nor probes the embedder.
+ (when (semantic.util/semantic-search-active?)
+ ;; Acquire the datasource inside the try, so a malformed MB_PGVECTOR_DB_URL reads as degraded instead of
+ ;; throwing out of the check.
+ (let [active (try
+ (let [pgvector (semantic.env/get-pgvector-datasource!)
+ index-metadata (semantic.env/get-index-metadata)]
+ {:pgvector pgvector
+ :state (semantic.index-metadata/get-active-index-state pgvector index-metadata)})
+ (catch InterruptedException e
+ (throw e))
+ (catch Exception e
+ {:error e}))]
+ (cond
+ (:error active)
+ (search.index-health/degraded (str "pgvector store unreachable: " (ex-message (:error active)) "."))
+
+ (nil? (:state active))
+ (search.index-health/degraded "No active semantic search index.")
+
+ :else
+ (let [{:keys [pgvector state]} active
+ index (:index state)
+ table-name (:table-name index)
+ stalled-at (-> state :metadata-row :indexer_stalled_at)
+ model-mismatch? (not= (:embedding-model index) (semantic.env/get-configured-embedding-model))
+ embedder-problem (when-not model-mismatch? (embedding-health/embedding-problem))
+ problems (cond-> []
+ (not (active-index-queryable? pgvector table-name))
+ (conj "active index table not queryable")
+
+ (not (required-hnsw-index-present? pgvector index))
+ (conj "required HNSW index is missing")
+
+ (some? stalled-at)
+ (conj (str "indexer stalled since " stalled-at))
+
+ model-mismatch?
+ (conj "active index embedding model does not match configured model")
+
+ embedder-problem
+ (conj embedder-problem))]
+ (if (seq problems)
+ (search.index-health/degraded (str "Semantic search degraded: " (str/join "; " problems) "."))
+ (search.index-health/healthy "Semantic search index active and serving.")))))))
+
+(health-inspector/register-check! :semantic-search-index index-health-check)
+
+(defn- persist-index-check-on-breaker-change!
+ "Re-run and persist the semantic-search index check."
+ [_state]
+ (health-inspector/run-and-save-check! :semantic-search-index))
+
+(swap! semantic.embedding/embedder-circuit-state-change-hooks conj #'persist-index-check-on-breaker-change!)
+
+;;; ------------------------------------- semantic-search collectors ----------------------------------------
+
+(def ^:private staleness-warn-seconds (* 60 60)) ; 1h -- indexer backlog under an hour reads healthy
+(def ^:private staleness-critical-seconds (* 6 60 60)) ; 6h -- a backlog this old = the indexer is stuck
+;; Absolute orphan counts. Repair clears garbage hourly, so a handful of in-flight orphans is normal; a large
+;; count means repair isn't keeping up. Tunable (promotable to settings alongside the staleness thresholds).
+(def ^:private garbage-warn-count 50)
+(def ^:private garbage-critical-count 5000)
+;; Repair runs hourly. Stop publishing its last snapshot after three missed runs, so an abandoned Quartz
+;; owner cannot leave a plausible-looking value behind indefinitely.
+(def ^:private repair-snapshot-max-age-seconds (* 3 60 60))
+
+(defn- active-index*
+ "The active semantic index and pgvector datasource, or nil when there is nothing applicable to measure."
+ []
+ (when (semantic.util/semantic-search-active?)
+ (try
+ (let [pgvector (semantic.env/get-pgvector-datasource!)
+ state (semantic.index-metadata/get-active-index-state pgvector (semantic.env/get-index-metadata))]
+ (when (and state
+ (active-index-queryable? pgvector (-> state :index :table-name)))
+ {:pgvector pgvector, :state state}))
+ (catch InterruptedException e
+ (throw e))
+ (catch Exception e
+ (log/debug e "semantic active-index lookup failed for a metric collector")
+ nil))))
+
+(def ^:private active-index
+ "Cached [[active-index*]] so collectors and the repair reporter can share the lookup."
+ (memoize/ttl active-index* :ttl/threshold (* 30 1000)))
+
+(defn- scalar-row [pgvector sql]
+ (jdbc/execute-one! pgvector sql {:builder-fn jdbc.rs/as-unqualified-lower-maps}))
+
+(defn- fresh-repair-age?
+ [age]
+ (and (number? age)
+ (<= 0.0 (double age) repair-snapshot-max-age-seconds)))
+
+(defn- repair-snapshot-age
+ "Age of `snapshot-at` in seconds, measured exclusively on the pgvector clock."
+ [pgvector snapshot-at]
+ (:repair_age
+ (scalar-row pgvector
+ ["SELECT EXTRACT(EPOCH FROM (clock_timestamp() - CAST(? AS timestamptz))) AS repair_age"
+ snapshot-at])))
+
+(defn- semantic-coverage []
+ (when-let [{:keys [pgvector state]} (active-index)]
+ (when-let [snapshot-at (-> state :metadata-row :repair_snapshot_at)]
+ (let [table (semantic.util/quote-table (-> state :index :table-name))
+ gate (semantic.util/quote-table (:gate-table-name (semantic.env/get-index-metadata)))
+ ;; The repair age and both populations share one pgvector statement snapshot. Tombstones leave
+ ;; the expected population immediately; stale index rows remain garbage's responsibility.
+ {:keys [indexed expected repair_age]}
+ (scalar-row pgvector
+ [(format (str "WITH observed AS (SELECT clock_timestamp() AS at) "
+ "SELECT count(*) FILTER (WHERE EXISTS "
+ "(SELECT 1 FROM %s i "
+ "WHERE i.model = g.model AND i.model_id = g.model_id)) AS indexed, "
+ "count(*) AS expected, "
+ "EXTRACT(EPOCH FROM ((SELECT at FROM observed) - "
+ "CAST(? AS timestamptz))) AS repair_age "
+ "FROM %s g WHERE g.document_hash IS NOT NULL")
+ table gate)
+ snapshot-at])]
+ (when (fresh-repair-age? repair_age)
+ (search.index-health/coverage-result (or indexed 0) (or expected 0)))))))
+
+(defn- semantic-staleness []
+ (when-let [{:keys [pgvector state]} (active-index)]
+ (let [index-metadata (semantic.env/get-index-metadata)
+ gate (semantic.util/quote-table (:gate-table-name index-metadata))
+ {:keys [id indexer_last_seen indexer_last_seen_id]} (:metadata-row state)
+ ;; Oldest gate change past the indexer watermark. Content-hash-based: the gate suppresses no-op
+ ;; updated_at bumps, so this is real pending work, not spurious timestamp drift. Age is computed on
+ ;; the pgvector clock to match gated_at (set via clock_timestamp).
+ ;; Composite (gated_at, id) comparison, matching the gate poll's consumption order -- a
+ ;; timestamp-only bound would hide pending rows that share the watermark timestamp. A nil watermark
+ ;; (indexer never ran) reads everything as pending; a nil id treats same-timestamp rows as pending
+ ;; ('' sorts before every real gate id).
+ gate-row (scalar-row pgvector
+ [(format "WITH observed AS (SELECT clock_timestamp() AS at)
+ SELECT count(*) AS pending,
+ EXTRACT(EPOCH FROM ((SELECT at FROM observed) - min(gated_at))) AS age,
+ (SELECT at FROM observed) AS observed_at
+ FROM %s
+ WHERE (gated_at, id) > (COALESCE(?, '-infinity'::timestamptz), COALESCE(?, ''))"
+ gate)
+ indexer_last_seen indexer_last_seen_id])
+ dlq-row (if (semantic.dlq/dlq-table-exists? pgvector index-metadata id)
+ (let [table (semantic.util/quote-table
+ (name (semantic.dlq/dlq-table-name-kw index-metadata id)))]
+ (scalar-row pgvector
+ [(format "SELECT count(*) AS pending,
+ EXTRACT(EPOCH FROM (CAST(? AS timestamptz) - min(d.error_gated_at))) AS age
+ FROM %s d
+ JOIN %s g ON g.id = d.gate_id
+ AND g.gated_at = d.error_gated_at"
+ table gate)
+ (:observed_at gate-row)]))
+ {:pending 0, :age nil})
+ gate-pending (or (:pending gate-row) 0)
+ dlq-pending (or (:pending dlq-row) 0)
+ ages (keep :age [gate-row dlq-row])
+ age (when (seq ages) (apply max (map double ages)))
+ detail (cond
+ (and (pos? gate-pending) (pos? dlq-pending))
+ (format "%d change(s) in the indexer backlog; %d failed change(s) awaiting DLQ retry."
+ gate-pending dlq-pending)
+
+ (pos? gate-pending)
+ (format "%d change(s) in the indexer backlog." gate-pending)
+
+ (pos? dlq-pending)
+ (format "%d failed change(s) awaiting DLQ retry." dlq-pending))]
+ (if (some #(neg? (double %)) ages)
+ (search.index-health/degraded "Index staleness unavailable: database clock precedes a pending change.")
+ (search.index-health/staleness-result
+ age staleness-warn-seconds staleness-critical-seconds detail)))))
+
+(defn- semantic-garbage
+ ;; The repair job writes a time-bounded snapshot to shared index metadata. Every node therefore publishes
+ ;; the same value even when Quartz moves the job between nodes.
+ []
+ (when-let [{:keys [pgvector state]} (active-index)]
+ (let [{:keys [repair_orphan_count repair_snapshot_at]} (:metadata-row state)]
+ (when (and (some? repair_orphan_count)
+ repair_snapshot_at
+ (fresh-repair-age? (repair-snapshot-age pgvector repair_snapshot_at)))
+ (search.index-health/garbage-result
+ repair_orphan_count garbage-warn-count garbage-critical-count)))))
+
+(search.index-health/register-index-check! :semantic-search :coverage semantic-coverage)
+
+(search.index-health/register-index-check! :semantic-search :staleness semantic-staleness)
+
+(def ^:private semantic-garbage-measure
+ "The semantic-search garbage descriptor. Holding it lets [[report-repair-metrics!]] refresh the metric
+ when a repair pushes a new count instead of waiting for the next collector cycle."
+ (search.index-health/register-index-check! :semantic-search :garbage semantic-garbage))
+
+(defn- clear-active-index-cache! []
+ (memoize/memo-clear! active-index))
+
+(defn- store-repair-metrics!
+ [index-id {:keys [orphan-count snapshot-at]}]
+ (let [pgvector (semantic.env/get-pgvector-datasource!)
+ index-metadata (semantic.env/get-index-metadata)
+ index-id (or index-id
+ (-> (semantic.index-metadata/get-active-index-state pgvector index-metadata)
+ :metadata-row
+ :id))
+ metadata-table (-> index-metadata :metadata-table-name semantic.util/quote-table)]
+ (when index-id
+ (jdbc/execute-one!
+ pgvector
+ [(format (str "UPDATE %s SET "
+ "repair_orphan_count = ?, "
+ "repair_snapshot_at = ? "
+ "WHERE id = ?")
+ metadata-table)
+ orphan-count snapshot-at index-id])
+ (clear-active-index-cache!))))
+
+(defn report-repair-metrics!
+ "Publish the repair job's stale-orphan count through shared index metadata.
+ `index-id` pins the snapshot to the index that repair measured; `:snapshot-at` is the time captured before
+ reading the canonical document stream. Nil counts invalidate the measure. Reporting failures do not fail
+ repair; interruption still propagates."
+ ([counts]
+ (report-repair-metrics! nil counts))
+ ([index-id counts]
+ (try
+ (store-repair-metrics! index-id counts)
+ (search.index-health/refresh-index-check! semantic-garbage-measure)
+ (catch InterruptedException e
+ (throw e))
+ (catch Exception e
+ (log/warn e "Failed to report semantic repair metrics")))))
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/index_metadata.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/index_metadata.clj
index 67aa335c3de..d68d7c3cf61 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/index_metadata.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/index_metadata.clj
@@ -38,7 +38,9 @@
[:indexer_last_seen :timestamp-with-time-zone :null]
[:indexer_last_seen_id :text :null]
[:indexer_last_seen_hash :text :null]
- [:indexer_stalled_at :timestamp-with-time-zone :null]]
+ [:indexer_stalled_at :timestamp-with-time-zone :null]
+ [:repair_orphan_count :bigint :null]
+ [:repair_snapshot_at :timestamp-with-time-zone :null]]
:control
[[:id :bigint [:primary-key]] ;; not auto-inc, only one row - still useful to ensure only one row when inserting.
@@ -118,6 +120,17 @@
(sql.helpers/with-columns schema)
(sql/format :quoted true))))
+(defn ensure-health-metric-columns!
+ "Add the non-destructive health-metric columns to an existing metadata table."
+ [pgvector {:keys [metadata-table-name]}]
+ (when (semantic.util/table-exists? pgvector metadata-table-name)
+ (jdbc/execute!
+ pgvector
+ [(format (str "ALTER TABLE %s "
+ "ADD COLUMN IF NOT EXISTS repair_orphan_count bigint NULL, "
+ "ADD COLUMN IF NOT EXISTS repair_snapshot_at timestamptz NULL")
+ (semantic.util/quote-table metadata-table-name))])))
+
(comment
(create-index-metadata-table-if-not-exists-sql default-index-metadata)
(create-control-table-if-not-exists-sql default-index-metadata)
@@ -144,7 +157,7 @@
nil))
(defn create-tables-if-not-exists!
- "Creates the metadata and control tables if they do not already exist"
+ "Create the metadata, control, and gate tables if needed."
[pgvector index-metadata]
(let [{:keys [metadata-table-name control-table-name gate-table-name version]} index-metadata]
;; **note** we do not currently deal with version mismatches as there is only one version.
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/init.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/init.clj
index 07908499c6c..61d95f9e33f 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/init.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/init.clj
@@ -2,6 +2,7 @@
(:require
[metabase-enterprise.semantic-search.embedding-health]
[metabase-enterprise.semantic-search.events]
+ [metabase-enterprise.semantic-search.health]
[metabase-enterprise.semantic-search.settings]
[metabase-enterprise.semantic-search.task.index-cleanup]
[metabase-enterprise.semantic-search.task.index-repair]
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/repair.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/repair.clj
index da233f0d664..122071bd44a 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/repair.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/repair.clj
@@ -37,31 +37,27 @@
(log/debugf "Populated repair table with %d document records" (count repair-records)))))
(defn find-lost-deletes
- "Performs an anti-join to find documents that exist in the gate table
- but are not in the repair table. These represent lost deletes."
+ "Return gate documents absent from the repair table, representing lost deletes.
+
+ Throws when the anti-join fails so the repair cannot publish a snapshot from incomplete evidence."
[pgvector gate-table-name repair-table-name]
- (try
- (let [column semantic.util/column-keyword
- anti-join-sql (-> (sql.helpers/select :model :model_id)
- (sql.helpers/from (keyword gate-table-name))
- (sql.helpers/where [:not [:exists
- (-> (sql.helpers/select 1)
- (sql.helpers/from (keyword repair-table-name))
- (sql.helpers/where [:and
- [:= (column repair-table-name "model")
- (column gate-table-name "model")]
- [:= (column repair-table-name "model_id")
- (column gate-table-name "model_id")]]))]])
- (sql/format :quoted true))
- results (jdbc/execute! pgvector anti-join-sql {:builder-fn jdbc.rs/as-unqualified-lower-maps})]
- (log/infof "Found %d documents in gate table that should be deleted" (count results))
- results)
- ;; TODO (Chris 2026-07-13) -- swallowing the error reads as "no lost deletes", so a transient anti-join
- ;; failure silently skips tombstoning that run. Tolerated because the next repair pass retries; revisit
- ;; if lost deletes start slipping through.
- (catch Exception e
- (log/errorf e "Error finding lost deletes between gate table %s and repair table %s"
- gate-table-name repair-table-name))))
+ (let [column semantic.util/column-keyword
+ anti-join-sql (sql/format
+ {:select [:model :model_id]
+ :from [(keyword gate-table-name)]
+ :where [:not
+ [:exists
+ {:select [1]
+ :from [(keyword repair-table-name)]
+ :where [:and
+ [:= (column repair-table-name "model")
+ (column gate-table-name "model")]
+ [:= (column repair-table-name "model_id")
+ (column gate-table-name "model_id")]]}]]}
+ :quoted true)
+ results (jdbc/execute! pgvector anti-join-sql {:builder-fn jdbc.rs/as-unqualified-lower-maps})]
+ (log/infof "Found %d documents in gate table that should be deleted" (count results))
+ results))
(defn find-lost-deletes-by-model
"Finds lost deletes and groups them by model for easier processing.
@@ -73,6 +69,75 @@
(group-by :model)
(m/map-vals #(map :model_id %)))))
+(defn count-stale-orphans
+ "Count rows in the active index table whose `(model, model_id)` is absent from the repair table (the current
+ candidate set) AND has no live or pending gate row -- garbage whose gated delete the indexer has already
+ consumed yet whose index row still stands. `metadata-row` supplies the indexer's composite
+ `(indexer_last_seen, indexer_last_seen_id)` consumption watermark; only tombstones at/behind it count, so
+ in-flight deletes and current DLQ retries (the staleness metric's backlog) don't read as garbage.
+ Callers run this BEFORE the current repair's own gate-deletes, for the same reason: a freshly found lost
+ delete would otherwise push a garbage spike that stands until the next hourly repair.
+ Returns nil if the count query fails: this feeds only the garbage health metric, so a metric-read blip
+ must not fail the repair run whose real work already committed."
+ [pgvector index-table-name gate-table-name repair-table-name dlq-table-name metadata-row]
+ ;; Unlike find-lost-deletes' gate-based anti-join, measuring on the index table excludes retained
+ ;; tombstones for rows the indexer already removed, which would keep the count inflated until tombstone
+ ;; cleanup runs.
+ (try
+ (let [{:keys [indexer_last_seen indexer_last_seen_id]} metadata-row
+ count-sql (sql/format
+ {:select [[:%count.* :n]]
+ :from [[(keyword index-table-name) :i]]
+ :where [:and
+ [:not
+ [:exists
+ {:select [1]
+ :from [[(keyword repair-table-name) :r]]
+ :where [:and
+ [:= :r.model :i.model]
+ [:= :r.model_id :i.model_id]]}]]
+ ;; no live gate row (document_hash NULL or row gone) and no pending tombstone the
+ ;; indexer hasn't consumed yet ((gated_at, id) past the watermark). A nil watermark
+ ;; (indexer never ran) reads all tombstones as pending; a nil watermark id treats
+ ;; same-timestamp tombstones as pending ('' sorts before every real gate id) --
+ ;; conservative in both cases, no false spike.
+ [:not
+ [:exists
+ {:select [1]
+ :from [[(keyword gate-table-name) :g]]
+ :where [:and
+ [:= :g.model :i.model]
+ [:= :g.model_id :i.model_id]
+ [:or
+ [:!= :g.document_hash nil]
+ [:> [:composite :g.gated_at :g.id]
+ [:composite
+ [:coalesce [:lift indexer_last_seen]
+ [:raw "'-infinity'::timestamptz"]]
+ [:coalesce [:lift indexer_last_seen_id] ""]]]]]}]]
+ ;; A failed delete may sit behind the watermark: stalled-mode advances it after
+ ;; moving the failure to the DLQ. The current gate generation still owns that retry,
+ ;; so it is pending work rather than garbage.
+ [:not
+ [:exists
+ {:select [1]
+ :from [[(keyword dlq-table-name) :d]]
+ :join [[(keyword gate-table-name) :dg]
+ [:and
+ [:= :dg.id :d.gate_id]
+ [:= :dg.gated_at :d.error_gated_at]]]
+ :where [:and
+ [:= :dg.model :i.model]
+ [:= :dg.model_id :i.model_id]]}]]]}
+ :quoted true)]
+ (or (:n (jdbc/execute-one! pgvector count-sql {:builder-fn jdbc.rs/as-unqualified-lower-maps})) 0))
+ (catch InterruptedException e
+ (throw e))
+ (catch Exception e
+ (log/errorf e "Error counting stale orphans in index table %s against repair table %s"
+ index-table-name repair-table-name)
+ nil)))
+
(defn- create-repair-table!
"Creates an empty temporary table for tracking documents during index repair."
[pgvector repair-table-name]
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/task/index_repair.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/task/index_repair.clj
index e2618064baf..f699e1e34d0 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/task/index_repair.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/task/index_repair.clj
@@ -5,6 +5,7 @@
[clojurewerkz.quartzite.schedule.cron :as cron]
[clojurewerkz.quartzite.triggers :as triggers]
[metabase-enterprise.semantic-search.core :as semantic-search.core]
+ [metabase-enterprise.semantic-search.health :as semantic.health]
[metabase-enterprise.semantic-search.util :as semantic.u]
[metabase.lib-be.core :as lib-be]
[metabase.search.ingestion :as search.ingestion]
@@ -18,20 +19,30 @@
(def ^:private repair-job-key (jobs/key "metabase.task.semantic-index-repair.job"))
(def ^:private repair-trigger-key (triggers/key "metabase.task.semantic-index-repair.trigger"))
+(defn- repair-index! []
+ (try
+ (log/info "Starting semantic search index repair")
+ ;; Keep the cache bound while the document eduction is realized inside repair-index!.
+ (lib-be/with-metadata-provider-cache
+ (let [{:keys [index-id orphans snapshot-at]}
+ (semantic-search.core/repair-index! (search.ingestion/searchable-documents))]
+ (semantic.health/report-repair-metrics! index-id {:orphan-count orphans
+ :snapshot-at snapshot-at})))
+ (log/info "Completed semantic search index repair")
+ (catch InterruptedException e
+ (semantic.health/report-repair-metrics! nil)
+ (throw e))
+ (catch Exception e
+ ;; Invalidate the shared snapshot when the producing job fails.
+ (semantic.health/report-repair-metrics! nil)
+ (log/error e "Failed to complete semantic search index repair"))))
+
(task/defjob ^{DisallowConcurrentExecution true
:doc "Runs repair-index! to maintain semantic search consistency"}
SemanticIndexRepair [_ctx]
(when (semantic.u/semantic-search-active?)
(log/with-context {:quartz-job-type 'SemanticIndexRepair}
- (try
- (log/info "Starting semantic search index repair")
- ;; The document eduction is realized inside repair-index!, so the metadata-provider cache must be
- ;; bound around the whole call for spec :fn attrs that touch metadata to benefit from it.
- (lib-be/with-metadata-provider-cache
- (semantic-search.core/repair-index! (search.ingestion/searchable-documents)))
- (log/info "Completed semantic search index repair")
- (catch Exception e
- (log/error e "Failed to complete semantic search index repair"))))))
+ (repair-index!))))
(defmethod task/init! ::SemanticIndexRepair [_]
(when (semantic.u/semantic-search-configured?)
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/util.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/util.clj
index 839790af25b..37e19af8307 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/util.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/util.clj
@@ -25,6 +25,15 @@
[table-name]
(second (qualified-table-parts table-name)))
+(defn quote-table
+ "Quote a possibly schema-qualified `table-name` for raw SQL, quoting schema and table separately so it
+ renders as \"schema\".\"table\" rather than one identifier with a literal dot."
+ [table-name]
+ (let [[schema table] (qualified-table-parts table-name)]
+ (if schema
+ (str (quote-ident schema) "." (quote-ident table))
+ (quote-ident table))))
+
(defn column-keyword
"A `table.column` reference as a dotted-name keyword, not a namespaced one.
HoneySQL quotes a keyword's namespace as one identifier, so a schema-qualified table there renders as a
@@ -69,6 +78,26 @@
{:builder-fn jdbc.rs/as-unqualified-lower-maps})
(:index_exists false))))
+(defn index-ready?
+ "Whether `index-name` is present and marked ready and valid by PostgreSQL. An in-progress or abandoned
+ concurrent build is not ready. A schema-qualified name matches only within its schema."
+ [pgvector index-name]
+ (let [[schema index] (qualified-table-parts index-name)]
+ (-> (jdbc/execute-one! pgvector
+ (if schema
+ [(str "SELECT exists (SELECT 1 FROM pg_class i "
+ "JOIN pg_namespace n ON n.oid = i.relnamespace "
+ "JOIN pg_index x ON x.indexrelid = i.oid "
+ "WHERE n.nspname = ? AND i.relname = ? "
+ "AND x.indisready AND x.indisvalid) AS index_ready")
+ schema index]
+ [(str "SELECT exists (SELECT 1 FROM pg_class i "
+ "JOIN pg_index x ON x.indexrelid = i.oid "
+ "WHERE i.relname = ? AND x.indisready AND x.indisvalid) AS index_ready")
+ index])
+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})
+ (:index_ready false))))
+
(defn semantic-search-configured?
"Whether to schedule the semantic-search Quartz jobs at startup.
True when the `:semantic-search` feature is present and a pgvector store might exist: a dedicated
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/core_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/core_test.clj
index f6c64514d7a..b189edbe209 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/core_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/core_test.clj
@@ -4,7 +4,9 @@
[metabase-enterprise.semantic-search.core :as semantic.core]
[metabase-enterprise.semantic-search.embedding :as semantic.embedding]
[metabase-enterprise.semantic-search.env :as semantic.env]
+ [metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata]
[metabase-enterprise.semantic-search.pgvector-api :as semantic.pgvector-api]
+ [metabase-enterprise.semantic-search.repair :as semantic.repair]
[metabase-enterprise.semantic-search.test-util :as semantic.tu]
[metabase-enterprise.semantic-search.util :as semantic.util]
[metabase.analytics-interface.core :as analytics]
@@ -16,6 +18,8 @@
[metabase.search.settings :as search.settings]
[metabase.test :as mt]))
+(set! *warn-on-reflection* true)
+
(use-fixtures :once #'semantic.tu/once-fixture)
(deftest supported?-requires-an-embedder-test
@@ -30,6 +34,32 @@
(constantly semantic.tu/mock-embedding-model)]
(is (true? (semantic.core/supported?))))))))
+(deftest ^:sequential repair-snapshot-precedes-canonical-document-read-test
+ (let [events (atom [])
+ active-state (atom [nil {:metadata-row {:id 17}}])
+ documents (map (fn [document]
+ (swap! events conj :document-read)
+ document)
+ [{}])]
+ (mt/with-premium-features #{:semantic-search}
+ (mt/with-dynamic-fn-redefs
+ [semantic.core/capture-repair-snapshot-at (fn [_]
+ (swap! events conj :snapshot)
+ ::snapshot-at)
+ semantic.env/get-pgvector-datasource! (constantly ::pgvector)
+ semantic.env/get-index-metadata (constantly ::index-metadata)
+ semantic.env/get-configured-embedding-model (constantly ::embedding-model)
+ semantic.index-metadata/get-active-index-state (fn [& _]
+ (let [state (first @active-state)]
+ (swap! active-state subvec 1)
+ state))
+ semantic.pgvector-api/init-semantic-search! (fn [& _])
+ semantic.repair/with-repair-table! (fn [_ _ f] (f "repair"))
+ semantic.pgvector-api/gate-updates! (fn [_ _ docs & _] (dorun docs))]
+ (is (= {:index-id 17, :orphans 0, :snapshot-at ::snapshot-at}
+ (semantic.core/repair-index! documents))))
+ (is (= [:snapshot :document-read] @events)))))
+
(deftest fallback-engine-available-with-semantic-test
(mt/with-premium-features #{:semantic-search}
(mt/with-temporary-setting-values [search.settings/search-engine "semantic"]
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_circuit_breaker_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_circuit_breaker_test.clj
index 9c09f1afc6e..5aa517beb51 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_circuit_breaker_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_circuit_breaker_test.clj
@@ -3,8 +3,12 @@
[clj-http.client :as http]
[clojure.test :refer :all]
[diehard.circuit-breaker :as dh.cb]
+ ;; loaded for side effects: the health namespaces register the breaker state-change hooks that
+ ;; state-change-persists-affected-checks-test exercises
+ [metabase-enterprise.entity-retrieval.health]
[metabase-enterprise.semantic-search.embedding :as semantic.embedding]
[metabase-enterprise.semantic-search.embedding-health :as embedding-health]
+ [metabase-enterprise.semantic-search.health]
[metabase-enterprise.semantic-search.models.token-tracking :as token-tracking]
[metabase-enterprise.semantic-search.settings :as semantic.settings]
[metabase.analytics-interface.core :as analytics]
@@ -303,6 +307,19 @@
(#'embedding-health/request-recovery-on-open! :open)
(is (= 1 @requests)))))
+(deftest ^:sequential state-change-persists-affected-checks-test
+ (testing "a breaker state transition runs the registered hooks, persisting both embedder-dependent health
+ checks immediately"
+ (let [persisted (atom [])
+ completed (CountDownLatch. 2)]
+ (mt/with-dynamic-fn-redefs
+ [health-inspector/run-and-save-check! (fn [check-name]
+ (swap! persisted conj check-name)
+ (.countDown completed))]
+ (#'semantic.embedding/on-embedder-circuit-state-change! :open)
+ (is (.await completed 5 TimeUnit/SECONDS) "state-change hooks did not finish")
+ (is (= #{:semantic-search-index :nlq-retrieval} (set @persisted)))))))
+
(deftest ^:sequential state-changes-run-serially-in-arrival-order-test
(testing "transitions queue through one agent: a later transition's hooks can't overtake an earlier
transition whose hook is still blocked (e.g. on a probe against a down service), so a slow
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/health_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/health_test.clj
new file mode 100644
index 00000000000..6eb212a8851
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/health_test.clj
@@ -0,0 +1,486 @@
+(ns metabase-enterprise.semantic-search.health-test
+ (:require
+ [clojure.core.memoize :as memoize]
+ [clojure.test :refer :all]
+ [java-time.api :as t]
+ [metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource]
+ [metabase-enterprise.semantic-search.dlq :as semantic.dlq]
+ [metabase-enterprise.semantic-search.embedding :as semantic.embedding]
+ [metabase-enterprise.semantic-search.embedding-health :as embedding-health]
+ [metabase-enterprise.semantic-search.env :as semantic.env]
+ [metabase-enterprise.semantic-search.health :as semantic.health]
+ [metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata]
+ [metabase-enterprise.semantic-search.settings :as semantic.settings]
+ [metabase-enterprise.semantic-search.util :as semantic.util]
+ [metabase.analytics-interface.core :as analytics]
+ [metabase.health-inspector.core :as health-inspector]
+ [metabase.search.index-health :as search.index-health]
+ [metabase.test :as mt]
+ [next.jdbc :as jdbc]
+ [next.jdbc.result-set :as jdbc.rs])
+ (:import
+ (java.util.concurrent CountDownLatch TimeUnit)))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private embedding-model
+ {:provider "ai-service", :model-name "test-model", :vector-dimensions 3})
+
+(def ^:private active-state
+ {:index {:table-name "index_test_table", :embedding-model embedding-model}
+ :metadata-row {:id 1
+ :indexer_stalled_at nil
+ :repair_orphan_count nil
+ :repair_snapshot_at nil}})
+
+(defn- do-with-enabled!
+ "Run `thunk` with the semantic engine active and the pgvector accessors stubbed to harmless values, so
+ each test only has to override the signal it cares about."
+ [thunk]
+ (mt/with-dynamic-fn-redefs
+ [semantic.util/semantic-search-active? (constantly true)
+ semantic.env/get-pgvector-datasource! (constantly ::pgvector)
+ semantic.env/get-index-metadata (constantly ::index-metadata)
+ semantic.env/get-configured-embedding-model (constantly embedding-model)
+ semantic.settings/semantic-search-vector-strategy (constantly :brute-force)
+ embedding-health/request-circuit-recovery! (constantly nil)]
+ (thunk)))
+
+(deftest not-enabled-is-omitted-test
+ (testing "an inactive engine (unlicensed, unconfigured, or another engine selected) -> nil (omitted, not a
+ misleading 'No active semantic search index' incident)"
+ (mt/with-dynamic-fn-redefs [semantic.util/semantic-search-active? (constantly false)]
+ (is (nil? (semantic.health/index-health-check))))))
+
+(deftest pgvector-unreachable-test
+ (testing "a pgvector error while resolving the active index reports as degraded, not as no-index"
+ (do-with-enabled!
+ (fn []
+ (mt/with-dynamic-fn-redefs
+ [semantic.index-metadata/get-active-index-state (fn [& _] (throw (ex-info "connection refused" {})))]
+ (is (=? {:health 0 :message #".*pgvector store unreachable: connection refused.*"}
+ (semantic.health/index-health-check))))))))
+
+(deftest health-probes-propagate-interruption-test
+ (testing "active-index lookup interruption propagates"
+ (do-with-enabled!
+ (fn []
+ (mt/with-dynamic-fn-redefs
+ [semantic.index-metadata/get-active-index-state (fn [& _] (throw (InterruptedException.)))]
+ (is (thrown? InterruptedException (semantic.health/index-health-check)))))))
+ (testing "queryability probe interruption propagates"
+ (mt/with-dynamic-fn-redefs [jdbc/execute-one! (fn [& _] (throw (InterruptedException.)))]
+ (is (thrown? InterruptedException
+ (#'semantic.health/active-index-queryable? ::pgvector "index_table"))))))
+
+(deftest metric-active-index-requires-queryable-table-test
+ (do-with-enabled!
+ (fn []
+ (mt/with-dynamic-fn-redefs
+ [semantic.index-metadata/get-active-index-state (constantly active-state)
+ semantic.health/active-index-queryable? (constantly false)]
+ (is (nil? (#'semantic.health/active-index*)))))))
+
+(deftest no-active-index-test
+ (testing "enabled but no active index -> degraded"
+ (do-with-enabled!
+ (fn []
+ (mt/with-dynamic-fn-redefs
+ [semantic.index-metadata/get-active-index-state (constantly nil)]
+ (is (=? {:health 0 :message #"No active semantic search index\."}
+ (semantic.health/index-health-check))))))))
+
+(deftest healthy-test
+ (testing "active + queryable + un-stalled + embedder reachable -> healthy 100"
+ (do-with-enabled!
+ (fn []
+ (mt/with-dynamic-fn-redefs
+ [semantic.index-metadata/get-active-index-state (constantly active-state)
+ semantic.health/active-index-queryable? (constantly true)
+ semantic.embedding/embedder-circuit-untrusted? (constantly false)
+ embedding-health/embedding-service-reachable? (constantly {:reachable? true :error nil})]
+ (is (=? {:health 100 :message #"Semantic search index active.*serving\."}
+ (semantic.health/index-health-check))))))))
+
+(deftest degraded-conditions-test
+ (do-with-enabled!
+ (fn []
+ (testing "a stalled indexer degrades and names the stall"
+ (mt/with-dynamic-fn-redefs
+ [semantic.index-metadata/get-active-index-state
+ (constantly (assoc-in active-state [:metadata-row :indexer_stalled_at] "2026-07-07"))
+ semantic.health/active-index-queryable? (constantly true)
+ semantic.embedding/embedder-circuit-untrusted? (constantly false)
+ embedding-health/embedding-service-reachable? (constantly {:reachable? true :error nil})]
+ (is (=? {:health 0 :message #".*indexer stalled since 2026-07-07.*"}
+ (semantic.health/index-health-check)))))
+ (testing "an unqueryable active table degrades"
+ (mt/with-dynamic-fn-redefs
+ [semantic.index-metadata/get-active-index-state (constantly active-state)
+ semantic.health/active-index-queryable? (constantly false)
+ semantic.embedding/embedder-circuit-untrusted? (constantly false)
+ embedding-health/embedding-service-reachable? (constantly {:reachable? true :error nil})]
+ (is (=? {:health 0 :message #".*active index table not queryable.*"}
+ (semantic.health/index-health-check)))))
+ (testing "an index-backed strategy without its HNSW index degrades"
+ (mt/with-dynamic-fn-redefs
+ [semantic.index-metadata/get-active-index-state (constantly active-state)
+ semantic.health/active-index-queryable? (constantly true)
+ semantic.settings/semantic-search-vector-strategy (constantly :hnsw)
+ semantic.util/index-ready? (constantly false)
+ semantic.embedding/embedder-circuit-untrusted? (constantly false)
+ embedding-health/embedding-service-reachable? (constantly {:reachable? true :error nil})]
+ (is (=? {:health 0 :message #".*required HNSW index is missing.*"}
+ (semantic.health/index-health-check)))))
+ (testing "an active index for a different embedding model degrades without probing the wrong model"
+ (mt/with-dynamic-fn-redefs
+ [semantic.index-metadata/get-active-index-state
+ (constantly (assoc-in active-state [:index :embedding-model :model-name] "old-model"))
+ semantic.health/active-index-queryable? (constantly true)
+ embedding-health/embedding-problem #(throw (ex-info "must not probe configured model" {}))]
+ (is (=? {:health 0 :message #".*active index embedding model does not match configured model.*"}
+ (semantic.health/index-health-check)))))
+ (testing "a non-closed breaker (open or half-open) still probes (so a recovered-but-idle embedder is
+ detectable) and degrades identically in both states"
+ (let [probed? (atom false)]
+ (mt/with-dynamic-fn-redefs
+ [semantic.index-metadata/get-active-index-state (constantly active-state)
+ semantic.health/active-index-queryable? (constantly true)
+ semantic.embedding/embedder-circuit-untrusted? (constantly true)
+ embedding-health/embedding-service-reachable? (fn [] (reset! probed? true) {:reachable? true})]
+ (is (=? {:health 0 :message #".*circuit open \(probe reachable.*"}
+ (semantic.health/index-health-check)))
+ (is (true? @probed?) "an open circuit still probes so recovery is detectable"))))
+ (testing "an open circuit with an unreachable probe reads the same as unreachable with a closed one --
+ state-independent wording, so a flapping breaker's re-persisted rows dedup instead of flooding"
+ (mt/with-dynamic-fn-redefs
+ [semantic.index-metadata/get-active-index-state (constantly active-state)
+ semantic.health/active-index-queryable? (constantly true)
+ semantic.embedding/embedder-circuit-untrusted? (constantly true)
+ embedding-health/embedding-service-reachable? (constantly {:reachable? false :error "boom"})]
+ (is (=? {:health 0 :message #".*embedding service unreachable: boom.*"}
+ (semantic.health/index-health-check)))))
+ (testing "an unreachable embedder degrades and names the error"
+ (mt/with-dynamic-fn-redefs
+ [semantic.index-metadata/get-active-index-state (constantly active-state)
+ semantic.health/active-index-queryable? (constantly true)
+ semantic.embedding/embedder-circuit-untrusted? (constantly false)
+ embedding-health/embedding-service-reachable? (constantly {:reachable? false :error "boom"})]
+ (is (=? {:health 0 :message #".*embedding service unreachable: boom.*"}
+ (semantic.health/index-health-check))))))))
+
+;; Tests the raw probe (`embedding-service-reachable?` is TTL-memoized, so calling it across cases would
+;; return a stale cached result).
+;; ^:sequential: redefs the process-global `get-configured-model` / `get-embedding` via with-redefs, so it
+;; must not run concurrently with other tests (e.g. entity-retrieval-available? reads get-configured-model).
+(deftest ^:sequential probe-embedding-service-test
+ (testing "a successful embed reads as reachable; the probe bypasses the breaker and silences snowplow"
+ (let [seen (atom nil)]
+ ;; get-embedding is a multimethod; with-dynamic-fn-redefs can't patch those, so with-redefs is required.
+ (with-redefs [semantic.embedding/get-configured-model (constantly {:model-name "m"})
+ semantic.embedding/get-embedding
+ (fn [_model _text opts]
+ (reset! seen (assoc opts :bypass semantic.embedding/*bypass-circuit-breaker*))
+ [0.1 0.2])]
+ (is (=? {:reachable? true :error nil} (#'embedding-health/probe-embedding-service)))
+ (is (=? {:bypass true :snowplow? false :record-tokens? false} @seen)
+ "the probe runs with the breaker bypassed, snowplow silenced, and tokens unrecorded"))))
+ (testing "an embedding failure reads as unreachable and captures the message"
+ (with-redefs [semantic.embedding/get-configured-model (constantly {:model-name "m"})
+ semantic.embedding/get-embedding (fn [& _] (throw (ex-info "connection refused" {})))]
+ (is (=? {:reachable? false :error "connection refused"}
+ (#'embedding-health/probe-embedding-service))))))
+
+(deftest ^:sequential breaker-transition-probe-cache-test
+ (testing "open and half-open transitions reuse the probe; recovery clears it"
+ (let [probes (atom 0)]
+ ;; get-embedding is a multimethod; with-dynamic-fn-redefs can't patch those, so with-redefs is required.
+ (with-redefs [semantic.embedding/get-configured-model (constantly {:model-name "m"})
+ semantic.embedding/get-embedding (fn [& _] (swap! probes inc) [0.1])]
+ (mt/with-dynamic-fn-redefs [health-inspector/run-and-save-check! (constantly nil)]
+ (memoize/memo-clear! @#'embedding-health/embedding-service-reachable?*)
+ (embedding-health/embedding-service-reachable?)
+ (is (= 1 @probes))
+ (#'embedding-health/clear-probe-cache-on-recovery! :open)
+ (#'embedding-health/clear-probe-cache-on-recovery! :half-open)
+ (embedding-health/embedding-service-reachable?)
+ (is (= 1 @probes) "an open<->half-open flap rides the cached probe")
+ (#'embedding-health/clear-probe-cache-on-recovery! :closed)
+ (embedding-health/embedding-service-reachable?)
+ (is (= 2 @probes) "recovery busts the cache so the persisted row can't be a stale 'unreachable'")
+ (memoize/memo-clear! @#'embedding-health/embedding-service-reachable?*))))))
+
+(deftest ^:sequential circuit-recovery-probe-test
+ (testing "recovery runs off-thread and supplies enough successful trials to close an untrusted circuit"
+ (let [calls (atom 0)
+ completed (CountDownLatch. 1)
+ state @#'embedding-health/recovery-state
+ before @state]
+ (try
+ (reset! state {:running? false, :last-start-ns nil})
+ (with-redefs [semantic.embedding/get-configured-model (constantly {:model-name "m"})
+ semantic.embedding/get-embedding
+ (fn [& _]
+ (when (= 2 (swap! calls inc))
+ (.countDown completed))
+ [0.1])]
+ (mt/with-dynamic-fn-redefs
+ [semantic.embedding/embedder-circuit-untrusted? #(< @calls 2)
+ embedding-health/schedule-recovery! future-call]
+ (is (nil? (embedding-health/request-circuit-recovery!)))
+ (is (.await completed 5 TimeUnit/SECONDS) "recovery trials did not finish")
+ (is (= 2 @calls))))
+ (finally
+ (reset! state before))))))
+
+(deftest semantic-coverage-tracks-live-gate-after-repair-snapshot-test
+ (when semantic.db.datasource/db-url
+ (let [pgvector (semantic.db.datasource/ensure-initialized-data-source!)
+ suffix (System/nanoTime)
+ index-t (str "coverage_index_" suffix)
+ gate-t (str "coverage_gate_" suffix)
+ snapshot-at (:snapshot_at
+ (jdbc/execute-one! pgvector ["SELECT clock_timestamp() AS snapshot_at"]
+ {:builder-fn jdbc.rs/as-unqualified-lower-maps}))
+ exec! #(jdbc/execute! pgvector [%])
+ coverage (fn []
+ (mt/with-dynamic-fn-redefs
+ [semantic.health/active-index
+ (constantly {:pgvector pgvector
+ :state (-> active-state
+ (assoc-in [:index :table-name] index-t)
+ (assoc-in [:metadata-row :repair_orphan_count] 0)
+ (assoc-in [:metadata-row :repair_snapshot_at]
+ snapshot-at))})
+ semantic.env/get-index-metadata (constantly {:gate-table-name gate-t})]
+ (#'semantic.health/semantic-coverage)))]
+ (try
+ (exec! (format "CREATE TABLE \"%s\" (model text, model_id text)" index-t))
+ (exec! (format "CREATE TABLE \"%s\" (model text, model_id text, document_hash text)" gate-t))
+ (exec! (format "INSERT INTO \"%s\" VALUES ('card','a','ha'),('card','b','hb')" gate-t))
+ (exec! (format "INSERT INTO \"%s\" VALUES ('card','a')" index-t))
+ (testing "coverage starts from the live gate and active index"
+ (is (=? {:value 0.5, :health 50, :message "1 of 2 expected items indexed (50%)."}
+ (coverage))))
+ (testing "a post-repair addition lowers coverage until it is indexed"
+ (exec! (format "INSERT INTO \"%s\" VALUES ('card','c','hc')" gate-t))
+ (is (=? {:health 33, :message "1 of 3 expected items indexed (33%)."} (coverage)))
+ (exec! (format "INSERT INTO \"%s\" VALUES ('card','c')" index-t))
+ (is (=? {:health 67, :message "2 of 3 expected items indexed (67%)."} (coverage))))
+ (testing "a tombstone leaves coverage immediately even while its index row lingers"
+ (exec! (format "UPDATE \"%s\" SET document_hash = NULL WHERE model_id = 'a'" gate-t))
+ (is (=? {:value 0.5, :health 50, :message "1 of 2 expected items indexed (50%)."}
+ (coverage))))
+ (testing "an orphan cannot inflate the numerator"
+ (exec! (format "INSERT INTO \"%s\" VALUES ('card','orphan')" index-t))
+ (is (=? {:value 0.5, :health 50, :message "1 of 2 expected items indexed (50%)."}
+ (coverage))))
+ (finally
+ (exec! (format "DROP TABLE IF EXISTS \"%s\"" index-t))
+ (exec! (format "DROP TABLE IF EXISTS \"%s\"" gate-t)))))))
+
+(deftest semantic-coverage-requires-a-fresh-database-aged-repair-test
+ (testing "no successful repair snapshot -> coverage is unknown without querying"
+ (mt/with-dynamic-fn-redefs
+ [semantic.health/active-index (constantly {:pgvector ::pgvector, :state active-state})
+ semantic.health/scalar-row #(throw (ex-info "must not query" {}))]
+ (is (nil? (#'semantic.health/semantic-coverage)))))
+ (testing "future-skewed and expired snapshots are unknown, never healthy"
+ (doseq [age [-1 10801]]
+ (let [queries (atom [])]
+ (mt/with-dynamic-fn-redefs
+ [semantic.health/active-index
+ (constantly {:pgvector ::pgvector
+ :state (assoc-in active-state [:metadata-row :repair_snapshot_at] ::snapshot-at)})
+ semantic.env/get-index-metadata (constantly {:gate-table-name "index_gate"})
+ semantic.health/scalar-row (fn [_ sql]
+ (swap! queries conj sql)
+ {:indexed 1, :expected 1, :repair_age age})]
+ (is (nil? (#'semantic.health/semantic-coverage))))
+ (is (= ::snapshot-at (-> @queries first second))
+ "the pgvector query receives the persisted database timestamp")
+ (is (re-find #"clock_timestamp" (-> @queries first first))
+ "repair age and coverage populations share one database statement")))))
+
+(deftest semantic-staleness-composite-watermark-test
+ ;; Hermetic: an ad-hoc gate table stands in, exercising the composite (gated_at, id) watermark SQL.
+ (when semantic.db.datasource/db-url
+ (let [pgvector (semantic.db.datasource/ensure-initialized-data-source!)
+ gate-t (str "staleness_gate_" (System/nanoTime))
+ wm-ts (t/offset-date-time "2026-01-01T12:00:00Z")
+ staleness (fn [metadata-row]
+ (mt/with-dynamic-fn-redefs
+ [semantic.health/active-index (constantly {:pgvector pgvector
+ :state {:metadata-row metadata-row}})
+ semantic.env/get-index-metadata (constantly {:gate-table-name gate-t
+ :index-table-qualifier "%s"})]
+ (#'semantic.health/semantic-staleness)))
+ backlog (fn [result] (or (some->> (:message result) (re-find #"(\d+) change\(s\)") second parse-long) 0))]
+ (try
+ (jdbc/execute! pgvector [(format "CREATE TABLE \"%s\" (id text, gated_at timestamptz)" gate-t)])
+ (jdbc/execute! pgvector [(format (str "INSERT INTO \"%s\" VALUES "
+ "('a', timestamptz '2026-01-01 11:00:00+00'), "
+ "('b', timestamptz '2026-01-01 12:00:00+00'), "
+ "('c', timestamptz '2026-01-01 12:00:00+00')")
+ gate-t)])
+ (testing "rows at the watermark timestamp with a later id are pending -- a timestamp-only bound
+ would hide them and read a stalled indexer as current"
+ (is (= 1 (backlog (staleness {:id 7, :indexer_last_seen wm-ts :indexer_last_seen_id "b"})))))
+ (testing "a nil watermark (indexer never ran) reads everything as pending"
+ (is (= 3 (backlog (staleness {:id 7, :indexer_last_seen nil :indexer_last_seen_id nil})))))
+ (testing "a timestamp-only watermark (no id) treats same-timestamp rows as pending"
+ (is (= 2 (backlog (staleness {:id 7, :indexer_last_seen wm-ts :indexer_last_seen_id nil})))))
+ (testing "a watermark past every row reads current"
+ (is (=? {:value 0, :health 100, :message #"Index current\."}
+ (staleness {:id 7
+ :indexer_last_seen (t/plus wm-ts (t/hours 1))
+ :indexer_last_seen_id ""}))))
+ (finally
+ (jdbc/execute! pgvector [(format "DROP TABLE IF EXISTS \"%s\"" gate-t)]))))))
+
+(deftest semantic-staleness-includes-active-dlq-test
+ (let [rows (atom [{:pending 0, :age nil, :observed_at ::observed-at}
+ {:pending 1, :age 25200}])
+ queries (atom [])]
+ (mt/with-dynamic-fn-redefs
+ [semantic.health/active-index
+ (constantly {:pgvector ::pgvector
+ :state {:metadata-row {:id 7, :indexer_last_seen nil, :indexer_last_seen_id nil}}})
+ semantic.env/get-index-metadata (constantly {:gate-table-name "index_gate"
+ :index-table-qualifier "%s"})
+ semantic.dlq/dlq-table-exists? (constantly true)
+ semantic.health/scalar-row (fn [_ sql]
+ (swap! queries conj sql)
+ (let [row (first @rows)]
+ (swap! rows subvec 1)
+ row))]
+ (is (=? {:value 25200
+ :health 0
+ :message #".*1 failed change\(s\) awaiting DLQ retry.*"}
+ (#'semantic.health/semantic-staleness)))
+ (is (re-find #"JOIN \"index_gate\" g ON g.id = d.gate_id\s+AND g.gated_at = d.error_gated_at"
+ (-> @queries second first))
+ "obsolete DLQ generations are excluded by matching the current gate timestamp")
+ (is (= ::observed-at (-> @queries second second))
+ "both backlog ages use the database time captured by the first query"))))
+
+(deftest semantic-staleness-rejects-negative-database-age-test
+ (mt/with-dynamic-fn-redefs
+ [semantic.health/active-index
+ (constantly {:pgvector ::pgvector
+ :state {:metadata-row {:id 7, :indexer_last_seen nil, :indexer_last_seen_id nil}}})
+ semantic.env/get-index-metadata (constantly {:gate-table-name "index_gate"
+ :index-table-qualifier "%s"})
+ semantic.dlq/dlq-table-exists? (constantly false)
+ semantic.health/scalar-row (constantly {:pending 1, :age -1, :observed_at ::observed-at})]
+ (is (=? {:health 0
+ :message "Index staleness unavailable: database clock precedes a pending change."}
+ (#'semantic.health/semantic-staleness)))))
+
+(deftest ^:sequential report-repair-metrics!-test
+ (mt/with-dynamic-fn-redefs [health-inspector/enabled? (constantly false)]
+ (testing "a stored snapshot refreshes the garbage gauge immediately"
+ (let [calls (atom [])
+ shared (atom nil)]
+ (mt/with-dynamic-fn-redefs
+ [semantic.health/store-repair-metrics! (fn [_index-id counts] (reset! shared counts))
+ semantic.health/active-index
+ #(let [{:keys [orphan-count]} @shared]
+ {:pgvector :x
+ :state (update active-state :metadata-row assoc
+ :repair_orphan_count orphan-count
+ :repair_snapshot_at (t/offset-date-time))})
+ semantic.health/repair-snapshot-age (constantly 0)
+ analytics/set-gauge! (fn [& args] (swap! calls conj (vec args)))]
+ (semantic.health/report-repair-metrics! {:orphan-count 3}))
+ (is (= [[:metabase-search/index-garbage-count {:index "semantic-search"} 3]] @calls))))
+ (testing "with no active index, the measure reads N/A instead of serving an old shared count"
+ (let [calls (atom [])]
+ (mt/with-dynamic-fn-redefs [semantic.health/store-repair-metrics! (constantly nil)
+ semantic.health/active-index (constantly nil)
+ analytics/set-gauge! (fn [& args] (swap! calls conj (vec args)))]
+ (semantic.health/report-repair-metrics! {:orphan-count 3})
+ (semantic.health/report-repair-metrics! nil))
+ (is (every? (fn [[_gauge _labels value]] (and (double? value) (Double/isNaN ^double value))) @calls)
+ "no real value reaches the gauge while there's no active index")))
+ (testing "an ordinary metric-sink error does not fail the repair job"
+ (mt/with-dynamic-fn-redefs [semantic.health/store-repair-metrics! (constantly nil)
+ semantic.health/active-index
+ (constantly {:pgvector :x
+ :state (update active-state :metadata-row assoc
+ :repair_orphan_count 3
+ :repair_snapshot_at (t/offset-date-time))})
+ semantic.health/repair-snapshot-age (constantly 0)
+ analytics/set-gauge! (fn [& _] (throw (ex-info "boom" {})))]
+ (is (nil? (semantic.health/report-repair-metrics! {:orphan-count 3})))))))
+
+(deftest invalid-repair-snapshot-age-is-omitted-test
+ (doseq [age [-1 10801]]
+ (mt/with-dynamic-fn-redefs
+ [semantic.health/active-index
+ (constantly {:pgvector :x
+ :state (update active-state :metadata-row assoc
+ :repair_orphan_count 3
+ :repair_snapshot_at ::snapshot-at)})
+ semantic.health/repair-snapshot-age (constantly age)]
+ (is (nil? (#'semantic.health/semantic-garbage))))))
+
+(deftest store-repair-metrics!-writes-exact-index-metadata-test
+ (let [calls (atom [])
+ snapshot-at (t/offset-date-time)]
+ (mt/with-dynamic-fn-redefs
+ [semantic.env/get-pgvector-datasource! (constantly ::pgvector)
+ semantic.env/get-index-metadata (constantly {:metadata-table-name "index_metadata"})
+ semantic.index-metadata/get-active-index-state #(throw (ex-info "explicit index id must win" {}))
+ semantic.health/clear-active-index-cache! (constantly nil)
+ jdbc/execute-one! (fn [& args] (swap! calls conj args))]
+ (#'semantic.health/store-repair-metrics! 17 {:orphan-count 7, :snapshot-at snapshot-at}))
+ (is (= [7 snapshot-at 17] (-> @calls first second rest vec)))
+ (is (re-find #"repair_orphan_count" (-> @calls first second first)))))
+
+(deftest store-repair-metrics!-resolves-current-index-without-the-read-cache-test
+ (let [calls (atom [])
+ snapshot-at (t/offset-date-time)]
+ (mt/with-dynamic-fn-redefs
+ [semantic.env/get-pgvector-datasource! (constantly ::pgvector)
+ semantic.env/get-index-metadata (constantly {:metadata-table-name "index_metadata"})
+ semantic.index-metadata/get-active-index-state (constantly active-state)
+ semantic.health/active-index (constantly nil)
+ semantic.health/clear-active-index-cache! (constantly nil)
+ jdbc/execute-one! (fn [& args] (swap! calls conj args))]
+ (#'semantic.health/store-repair-metrics! nil {:orphan-count 7, :snapshot-at snapshot-at}))
+ (is (= [7 snapshot-at 1] (-> @calls first second rest vec)))))
+
+(deftest report-repair-metrics!-propagates-interruption-test
+ (mt/with-dynamic-fn-redefs
+ [semantic.health/store-repair-metrics! (fn [& _] (throw (InterruptedException.)))]
+ (is (thrown? InterruptedException
+ (semantic.health/report-repair-metrics! {:orphan-count 3})))))
+
+(deftest ^:sequential refresh-clears-garbage-when-inactive-test
+ (testing "refresh NaNs a previously-emitted semantic garbage series when there's no active index, since no
+ repair push will clear it (the collector reads N/A)"
+ (let [calls (atom [])]
+ (mt/with-dynamic-fn-redefs [analytics/set-gauge! (fn [& args] (swap! calls conj (vec args)))]
+ (mt/with-dynamic-fn-redefs [health-inspector/enabled? (constantly false)]
+ ;; make the series live: a repair push while the feature was on
+ (mt/with-dynamic-fn-redefs
+ [semantic.health/store-repair-metrics! (constantly nil)
+ semantic.health/active-index
+ (constantly {:pgvector :x
+ :state (update active-state :metadata-row assoc
+ :repair_orphan_count 3
+ :repair_snapshot_at (t/offset-date-time))})]
+ (mt/with-dynamic-fn-redefs
+ [semantic.health/repair-snapshot-age (constantly 0)]
+ (semantic.health/report-repair-metrics! {:orphan-count 3})))
+ (reset! calls [])
+ (memoize/memo-clear! @#'semantic.health/active-index)
+ (mt/with-dynamic-fn-redefs [semantic.util/semantic-search-active? (constantly false)]
+ (search.index-health/refresh-search-index-metrics!))))
+ (is (some (fn [[gauge labels value]]
+ (and (= gauge :metabase-search/index-garbage-count)
+ (= labels {:index "semantic-search"})
+ (double? value) (Double/isNaN ^double value)))
+ @calls)
+ "the semantic garbage-count series is cleared with NaN"))))
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/migration_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/migration_test.clj
index de40a2b0254..f23671908fd 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/migration_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/migration_test.clj
@@ -300,6 +300,21 @@
[:= :table_name [:inline table-name]]
[:= :column_name [:inline column-name]]]}))))
+(deftest current-schema-version-adds-health-metric-columns-test
+ (mt/with-premium-features #{:semantic-search}
+ (semantic.tu/with-test-db-defaults!
+ (let [pgvector (semantic.env/get-pgvector-datasource!)
+ index-metadata semantic.index-metadata/default-index-metadata]
+ (semantic.db.connection/with-migrate-tx [tx]
+ (semantic.db.migration/maybe-migrate! tx {:index-metadata index-metadata}))
+ (jdbc/execute! pgvector [(str "ALTER TABLE index_metadata "
+ "DROP COLUMN repair_orphan_count, "
+ "DROP COLUMN repair_snapshot_at")])
+ (semantic.db.connection/with-migrate-tx [tx]
+ (semantic.db.migration/maybe-migrate! tx {:index-metadata index-metadata}))
+ (is (has-column?! pgvector "index_metadata" "repair_orphan_count"))
+ (is (has-column?! pgvector "index_metadata" "repair_snapshot_at"))))))
+
(deftest dynamic-schema-migration-test
(mt/with-premium-features #{:semantic-search}
(semantic.tu/with-test-db-defaults!
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/repair_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/repair_test.clj
index 72ca9a3f6be..7251f2a8150 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/repair_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/repair_test.clj
@@ -5,6 +5,7 @@
[honey.sql :as sql]
[java-time.api :as t]
[metabase-enterprise.semantic-search.core :as semantic.core]
+ [metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource]
[metabase-enterprise.semantic-search.embedding :as semantic.embedding]
[metabase-enterprise.semantic-search.env :as semantic.env]
[metabase-enterprise.semantic-search.index :as semantic.index]
@@ -72,11 +73,10 @@
(create-test-document "dashboard" 3 "Animal Stats")]]
(semantic.core/update-index! initial-docs)
(semantic.tu/index-all!)
- ;; Ensure repair-index! brings index into consistency with new documents et.
- ;; Note: card 2 and dashboard 3 are missing - should be deleted
- (let [modified-docs [(create-test-document "card" 1 "Dog Training Guide") ; existing - should remain
- (create-test-document "card" 4 "Bird Watching Tips") ; new - should be added
- (create-test-document "dashboard" 5 "Wildlife Dashboard")]] ; new - should be added
+ ;; Retain card 1, add card 4 and dashboard 5, and tombstone the missing card 2 and dashboard 3.
+ (let [modified-docs [(create-test-document "card" 1 "Dog Training Guide")
+ (create-test-document "card" 4 "Bird Watching Tips")
+ (create-test-document "dashboard" 5 "Wildlife Dashboard")]]
(semantic.core/repair-index! modified-docs)
(let [gate-contents (gate-table-contents pgvector gate-table)
new-card-entry (gate-entry-by-id gate-contents "card_4")
@@ -108,6 +108,75 @@
(let [gate-contents (gate-table-contents pgvector (:gate-table-name index-metadata))]
(is (some? (:document (gate-entry-by-id gate-contents "card_1"))))))))))))
+(deftest count-stale-orphans-test
+ ;; Hermetic: the fn only takes table names, so ad-hoc tables stand in for the index/gate/repair tables.
+ (when semantic.db.datasource/db-url
+ (let [pgvector (semantic.db.datasource/ensure-initialized-data-source!)
+ suffix (System/nanoTime)
+ index-t (str "count_orphans_index_" suffix)
+ gate-t (str "count_orphans_gate_" suffix)
+ repair-t (str "count_orphans_repair_" suffix)
+ dlq-t (str "count_orphans_dlq_" suffix)
+ exec! (fn [q] (jdbc/execute! pgvector [q]))
+ count! (fn [watermark]
+ (semantic.repair/count-stale-orphans pgvector index-t gate-t repair-t dlq-t watermark))
+ wm-ts (t/offset-date-time "2026-01-01T12:00:00Z")]
+ (try
+ (exec! (format "CREATE TABLE \"%s\" (model text, model_id text)" index-t))
+ (exec! (format (str "CREATE TABLE \"%s\" (id text, model text, model_id text, "
+ "document_hash text, gated_at timestamptz)")
+ gate-t))
+ (exec! (format "CREATE TABLE \"%s\" (model text, model_id text)" repair-t))
+ (exec! (format "CREATE TABLE \"%s\" (gate_id text, error_gated_at timestamptz)" dlq-t))
+ ;; candidate set: card 1 only
+ (exec! (format "INSERT INTO \"%s\" VALUES ('card','1')" repair-t))
+ ;; index rows: 1 = still a candidate, 2 = tombstone behind the watermark (stale), 3 = tombstone ahead
+ ;; of it (indexer backlog), 4 = gate row gone (survived tombstone cleanup), 5 = live gate row,
+ ;; 6/7 = tombstones AT the watermark timestamp with gate ids on either side of the watermark id
+ (exec! (format (str "INSERT INTO \"%s\" VALUES "
+ "('card','1'),('card','2'),('card','3'),('card','4'),('card','5'),"
+ "('card','6'),('card','7')")
+ index-t))
+ (exec! (format (str "INSERT INTO \"%s\" VALUES "
+ "('card_2', 'card','2', NULL, timestamptz '2026-01-01 11:00:00+00'), "
+ "('card_3', 'card','3', NULL, timestamptz '2026-01-01 13:00:00+00'), "
+ "('card_5', 'card','5', 'hash', timestamptz '2026-01-01 11:00:00+00'), "
+ "('card_6', 'card','6', NULL, timestamptz '2026-01-01 12:00:00+00'), "
+ "('card_7', 'card','7', NULL, timestamptz '2026-01-01 12:00:00+00')")
+ gate-t))
+ ;; card 2 is a current failed delete and remains pending; card 3's obsolete DLQ generation must not
+ ;; hide it once the watermark passes its current tombstone.
+ (exec! (format (str "INSERT INTO \"%s\" VALUES "
+ "('card_2', timestamptz '2026-01-01 11:00:00+00'), "
+ "('card_3', timestamptz '2026-01-01 10:00:00+00')")
+ dlq-t))
+ (testing "counts non-candidates whose tombstone the (gated_at, id) watermark has passed, or whose
+ gate row is gone"
+ ;; Watermark id between card_6 and card_7: cards 4 (gate row gone) and 6 (same timestamp, earlier
+ ;; id) count. Card 2 has a current DLQ retry; cards 3 and 7 remain ahead of the watermark.
+ (is (= 2 (count! {:indexer_last_seen wm-ts :indexer_last_seen_id "card_6a"}))))
+ (testing "a tombstone ahead of the watermark is in-flight backlog, not garbage"
+ (is (= 4 (count! {:indexer_last_seen (t/plus wm-ts (t/hours 2)) :indexer_last_seen_id ""}))
+ "once the watermark passes them, the same tombstones count"))
+ (testing "nil watermark (indexer never ran) counts no tombstones, only gate-row-gone orphans"
+ (is (= 1 (count! nil))))
+ (testing "a timestamp-only watermark (no id) treats same-timestamp tombstones as pending"
+ (is (= 1 (count! {:indexer_last_seen wm-ts}))
+ "cards 6/7 are pending by watermark and card 2 by DLQ; only gate-row-gone card 4 counts"))
+ (testing "a query failure returns nil rather than failing the repair run"
+ (is (nil? (semantic.repair/count-stale-orphans pgvector "no_such_table" gate-t repair-t dlq-t
+ {:indexer_last_seen wm-ts}))))
+ (finally
+ (exec! (format "DROP TABLE IF EXISTS \"%s\", \"%s\", \"%s\", \"%s\""
+ index-t gate-t repair-t dlq-t)))))))
+
+(deftest count-stale-orphans-propagates-interruption-test
+ (mt/with-dynamic-fn-redefs
+ [jdbc/execute-one! (fn [& _] (throw (InterruptedException.)))]
+ (is (thrown? InterruptedException
+ (semantic.repair/count-stale-orphans
+ ::pgvector "index" "gate" "repair" "dlq" {})))))
+
(deftest repair-table-cleanup-test
(testing "The repair table gets cleaned up properly at the end of a repair-index! job"
(mt/with-premium-features #{:semantic-search}
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/task/index_repair_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/task/index_repair_test.clj
new file mode 100644
index 00000000000..4873a49ddc6
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/task/index_repair_test.clj
@@ -0,0 +1,41 @@
+(ns metabase-enterprise.semantic-search.task.index-repair-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase-enterprise.semantic-search.core :as semantic-search.core]
+ [metabase-enterprise.semantic-search.health :as semantic.health]
+ [metabase-enterprise.semantic-search.repair :as semantic.repair]
+ [metabase-enterprise.semantic-search.task.index-repair :as index-repair]
+ [metabase.search.ingestion :as search.ingestion]
+ [metabase.test :as mt]
+ [next.jdbc :as jdbc]))
+
+(deftest successful-repair-reports-current-metric-snapshot-test
+ (let [reported (atom [])
+ snapshot-at ::snapshot-at]
+ (mt/with-dynamic-fn-redefs
+ [search.ingestion/searchable-documents (constantly [])
+ semantic-search.core/repair-index! (constantly {:index-id 11, :orphans 7, :snapshot-at snapshot-at})
+ semantic.health/report-repair-metrics! (fn [& args] (swap! reported conj args))]
+ (#'index-repair/repair-index!)
+ (is (= [[11 {:orphan-count 7, :snapshot-at snapshot-at}]] @reported)))))
+
+(deftest lost-delete-query-failure-invalidates-metric-snapshot-test
+ (let [reported (atom [])]
+ (mt/with-dynamic-fn-redefs
+ [search.ingestion/searchable-documents (constantly [])
+ jdbc/execute! (fn [& _] (throw (ex-info "anti-join failed" {})))
+ semantic-search.core/repair-index! (fn [& _]
+ (semantic.repair/find-lost-deletes
+ ::pgvector "gate" "repair"))
+ semantic.health/report-repair-metrics! (fn [& args] (swap! reported conj args))]
+ (#'index-repair/repair-index!)
+ (is (= [[nil]] @reported)))))
+
+(deftest interrupted-repair-invalidates-count-and-propagates-test
+ (let [reported (atom [])]
+ (mt/with-dynamic-fn-redefs
+ [search.ingestion/searchable-documents (constantly [])
+ semantic-search.core/repair-index! (fn [& _] (throw (InterruptedException.)))
+ semantic.health/report-repair-metrics! (fn [& args] (swap! reported conj args))]
+ (is (thrown? InterruptedException (#'index-repair/repair-index!)))
+ (is (= [[nil]] @reported)))))
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/util_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/util_test.clj
index 9cff3b055ee..733d2722240 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/util_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/util_test.clj
@@ -16,6 +16,13 @@
(is (= "\"semantic_search\"" (semantic.util/quote-ident "semantic_search")))
(is (= "\"weird\"\"name\"" (semantic.util/quote-ident "weird\"name")))))
+(deftest ^:parallel quote-table-test
+ (testing "a schema-qualified name quotes each part separately, not the whole thing as one identifier"
+ (is (= "\"semantic_search\".\"index_table_1\""
+ (semantic.util/quote-table "semantic_search.index_table_1"))))
+ (testing "an unqualified name is a single quoted identifier"
+ (is (= "\"index_table_1\"" (semantic.util/quote-table "index_table_1")))))
+
(deftest ^:parallel column-keyword-test
(testing "a dotted-name keyword (renders as separate identifiers), never a namespaced one"
(is (= :index_table_1.model (semantic.util/column-keyword "index_table_1" "model")))
@@ -26,6 +33,22 @@
(is (= :index_table_1.model
(semantic.util/conflict-target-column "semantic_search.index_table_1" "model")))))
+(deftest index-ready?-catalog-query-test
+ (let [queries (atom [])]
+ (mt/with-dynamic-fn-redefs
+ [jdbc/execute-one! (fn [_ query _]
+ (swap! queries conj query)
+ {:index_ready true})]
+ (is (true? (semantic.util/index-ready? ::pgvector "semantic_search.index_1")))
+ (is (true? (semantic.util/index-ready? ::pgvector "index_2"))))
+ (testing "both catalog queries require PostgreSQL to mark the index ready and valid"
+ (doseq [[statement] @queries]
+ (is (re-find #"x\.indisready" statement))
+ (is (re-find #"x\.indisvalid" statement))))
+ (testing "qualified names retain their schema constraint"
+ (is (= ["semantic_search" "index_1"] (rest (first @queries))))
+ (is (= ["index_2"] (rest (second @queries)))))))
+
(deftest catalog-lookups-schema-scoping-test
(testing "qualified names scope table/index existence checks to their schema; a same-named object in
another schema (e.g. the app db's public) must not satisfy them"