The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Add semantic search health checks (#78299)
Chris Truter1ba3a7c05048
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex e6d1d7c1a8c..08c75d27b46 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -3597,6 +3597,7 @@configconnection-poolevents+ health-inspectorlib-bellmmetabotdiff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/core.cljindex 8f6fa7028be..5b41441e212 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj@@ -3,6 +3,7 @@(:require[clojure.string :as str][medley.core :as m]+ [metabase-enterprise.semantic-search.dlq :as semantic.dlq][metabase-enterprise.semantic-search.embedders][metabase-enterprise.semantic-search.embedding :as semantic.embedding][metabase-enterprise.semantic-search.env :as semantic.env]@@ -17,6 +18,8 @@[metabase.search.engine :as search.engine][metabase.tracing.core :as tracing][metabase.util.log :as log]+ [next.jdbc :as jdbc]+ [next.jdbc.result-set :as jdbc.rs][potemkin :as p][toucan2.realize :as t2.realize]))@@ -35,6 +38,10 @@(defn- index-active? [pgvector index-metadata](boolean (semantic.index-metadata/get-active-index-state pgvector index-metadata)))+(defn- capture-repair-snapshot-at [pgvector]+ (:snapshot_at (jdbc/execute-one! pgvector ["SELECT clock_timestamp() AS snapshot_at"]+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})))+(defenterprise supported?"Enterprise implementation of semantic search engine support check.":feature :semantic-search@@ -184,17 +191,33 @@(defenterprise repair-index!"Brings the semantic search index into consistency with the provided document set.- Does not fully reinitialize the index, but will add missing documents and remove stale ones."+ Does not fully reinitialize the index, but will add missing documents and remove stale ones.+ Returns the repaired index ID, its stale-orphan count, and the pgvector timestamp captured before reading+ the canonical stream.":feature :semantic-search[searchable-documents](let [pgvector (semantic.env/get-pgvector-datasource!)- index-metadata (semantic.env/get-index-metadata)]- (if-not (index-active? pgvector index-metadata)+ index-metadata (semantic.env/get-index-metadata)+ snapshot-at (capture-repair-snapshot-at pgvector)+ active-state (semantic.index-metadata/get-active-index-state pgvector index-metadata)]+ (if-not active-state;; Semantic can become active at runtime (license applied, or added to additional-search-engines);; without init! ever having run; initializing here lets the periodic repair task backfill the index.(do(log/info "No active semantic index, initializing it instead of repairing")- (init! searchable-documents {}))+ (semantic.pgvector-api/init-semantic-search!+ pgvector index-metadata (semantic.env/get-configured-embedding-model) {})+ (semantic.repair/with-repair-table!+ pgvector+ index-metadata+ (fn [repair-table-name]+ (semantic.pgvector-api/gate-updates! pgvector index-metadata searchable-documents+ :repair-table repair-table-name)+ {:index-id (-> (semantic.index-metadata/get-active-index-state pgvector index-metadata)+ :metadata-row+ :id)+ :orphans 0+ :snapshot-at snapshot-at})))(semantic.repair/with-repair-table!pgvectorindex-metadata@@ -202,11 +225,27 @@;; Re-gate all provided documents, populating the repair table as we go(semantic.pgvector-api/gate-updates! pgvector index-metadata searchable-documents:repair-table repair-table-name)- ;; Find documents in the gate table that are not in the provided searchable-documents, and gate deletes for them- (when-let [ids-by-model (semantic.repair/find-lost-deletes-by-model pgvector (:gate-table-name index-metadata) repair-table-name)]- (doseq [[model ids] ids-by-model]- (log/infof "Repairing lost deletes for model %s: deleting %d documents" model (count ids))- (semantic.pgvector-api/gate-deletes! pgvector index-metadata model ids))))))))+ ;; Counted BEFORE this run's gate-deletes, so the lost deletes found below (in-flight cleanup the+ ;; indexer typically clears within minutes) don't read as a garbage spike that stands until the+ ;; next hourly repair; see [[semantic.repair/count-stale-orphans]] for the full contract.+ (let [orphans (semantic.repair/count-stale-orphans pgvector+ (-> active-state :index :table-name)+ (:gate-table-name index-metadata)+ repair-table-name+ (semantic.dlq/dlq-table-name-kw+ index-metadata+ (-> active-state :metadata-row :id))+ (:metadata-row active-state))]+ ;; Gate deletes for documents absent from the current searchable set.+ (when-let [ids-by-model+ (semantic.repair/find-lost-deletes-by-model+ pgvector (:gate-table-name index-metadata) repair-table-name)]+ (doseq [[model ids] ids-by-model]+ (log/infof "Repairing lost deletes for model %s: deleting %d documents" model (count ids))+ (semantic.pgvector-api/gate-deletes! pgvector index-metadata model ids)))+ {:index-id (-> active-state :metadata-row :id)+ :orphans orphans+ :snapshot-at snapshot-at}))))))(comment(update-index! [{:model "card"diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration.cljindex 15e3772f6d5..ce43fba805a 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration.clj@@ -88,7 +88,11 @@:else(log/infof "Database schema version (%d) is newer than code version (%d). Not performing migration."- db-version semantic.db.migration.impl/schema-version)))+ db-version semantic.db.migration.impl/schema-version))+ ;; Existing schema-version-2 databases predate these additive columns. This compatibility pass must also+ ;; run when the migration version already matches; newer schemas remain untouched by older code.+ (when (<= db-version semantic.db.migration.impl/schema-version)+ (semantic.db.migration.impl/ensure-schema-compatibility! tx opts)))nil)(defn- index-metadata-table-exists?diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration/impl.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration/impl.cljindex 27e0ee9f45b..cbbbd357d2a 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration/impl.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration/impl.clj@@ -66,6 +66,11 @@(semantic.index-metadata/create-tables-if-not-exists! tx index-metadata)(semantic.index-metadata/ensure-control-row-exists! tx index-metadata))+(defn ensure-schema-compatibility!+ "Apply non-destructive additions that existing databases at the current schema version also need."+ [tx {:keys [index-metadata]}]+ (semantic.index-metadata/ensure-health-metric-columns! tx index-metadata))+(def dynamic-schema-version"Code version of dynamic schema (the index_<provider>_<model>_<dims> index tables). If higher than what's found indb dynamic schema migration will be attempted."diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/health.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/health.cljnew file mode 100644index 00000000000..04ac4c6d72b--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/health.clj@@ -0,0 +1,298 @@+(ns metabase-enterprise.semantic-search.health+ "Health-inspector checks and index metrics for semantic search.++ Semantic search degrades silently -- a missing index, stalled indexer, or unreachable embedder just falls+ back to appdb search, unnoticed. This check surfaces that.++ :health can take the following values:+ - nil = not enabled (check omitted)+ - 0 = enabled but degraded (`:message` names the cause)+ - 100 = serving"+ (:require+ [clojure.core.memoize :as memoize]+ [clojure.string :as str]+ [metabase-enterprise.semantic-search.dlq :as semantic.dlq]+ [metabase-enterprise.semantic-search.embedding :as semantic.embedding]+ [metabase-enterprise.semantic-search.embedding-health :as embedding-health]+ [metabase-enterprise.semantic-search.env :as semantic.env]+ [metabase-enterprise.semantic-search.index :as semantic.index]+ [metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata]+ [metabase-enterprise.semantic-search.settings :as semantic.settings]+ [metabase-enterprise.semantic-search.util :as semantic.util]+ [metabase.health-inspector.core :as health-inspector]+ [metabase.search.config :as search.config]+ [metabase.search.index-health :as search.index-health]+ [metabase.util.log :as log]+ [next.jdbc :as jdbc]+ [next.jdbc.result-set :as jdbc.rs]))++(set! *warn-on-reflection* true)++(defn- active-index-queryable?+ "Whether a trivial `SELECT ... LIMIT 1` against the active index table succeeds -- i.e. pgvector is+ reachable and the table exists and is queryable."+ [pgvector table-name]+ (try+ (jdbc/execute-one! pgvector [(format "SELECT 1 FROM %s LIMIT 1" (semantic.util/quote-table table-name))])+ true+ (catch InterruptedException e+ (throw e))+ (catch Exception _ false)))++(defn- required-hnsw-index-present?+ "Whether the active index satisfies the configured default vector strategy."+ [pgvector index]+ (try+ (or (not (contains? search.config/hnsw-index-backed-strategies+ (semantic.settings/semantic-search-vector-strategy)))+ (semantic.util/index-ready?+ pgvector+ (semantic.index/schema-qualified-index-name index (semantic.index/hnsw-index-name index))))+ (catch InterruptedException e+ (throw e))+ (catch Exception _ false)))++(defn index-health-check+ "Health-inspector check for semantic search, registered as `:semantic-search-index`.+ Returns nil when the semantic engine isn't active on this instance (so the check is omitted), otherwise a+ `{:health :message}` map: healthy when the active index matches the configured model and is queryable,+ un-stalled, strategy-ready, and backed by a reachable embedder; degraded otherwise."+ []+ ;; Active, not merely available: an available-but-inactive engine (another engine selected) has no index+ ;; by design, and must not read as a standing "No active semantic search index" incident. Active folds in+ ;; the license and kill switch, so a disabled instance neither records runs nor probes the embedder.+ (when (semantic.util/semantic-search-active?)+ ;; Acquire the datasource inside the try, so a malformed MB_PGVECTOR_DB_URL reads as degraded instead of+ ;; throwing out of the check.+ (let [active (try+ (let [pgvector (semantic.env/get-pgvector-datasource!)+ index-metadata (semantic.env/get-index-metadata)]+ {:pgvector pgvector+ :state (semantic.index-metadata/get-active-index-state pgvector index-metadata)})+ (catch InterruptedException e+ (throw e))+ (catch Exception e+ {:error e}))]+ (cond+ (:error active)+ (search.index-health/degraded (str "pgvector store unreachable: " (ex-message (:error active)) "."))++ (nil? (:state active))+ (search.index-health/degraded "No active semantic search index.")++ :else+ (let [{:keys [pgvector state]} active+ index (:index state)+ table-name (:table-name index)+ stalled-at (-> state :metadata-row :indexer_stalled_at)+ model-mismatch? (not= (:embedding-model index) (semantic.env/get-configured-embedding-model))+ embedder-problem (when-not model-mismatch? (embedding-health/embedding-problem))+ problems (cond-> []+ (not (active-index-queryable? pgvector table-name))+ (conj "active index table not queryable")++ (not (required-hnsw-index-present? pgvector index))+ (conj "required HNSW index is missing")++ (some? stalled-at)+ (conj (str "indexer stalled since " stalled-at))++ model-mismatch?+ (conj "active index embedding model does not match configured model")++ embedder-problem+ (conj embedder-problem))]+ (if (seq problems)+ (search.index-health/degraded (str "Semantic search degraded: " (str/join "; " problems) "."))+ (search.index-health/healthy "Semantic search index active and serving.")))))))++(health-inspector/register-check! :semantic-search-index index-health-check)++(defn- persist-index-check-on-breaker-change!+ "Re-run and persist the semantic-search index check."+ [_state]+ (health-inspector/run-and-save-check! :semantic-search-index))++(swap! semantic.embedding/embedder-circuit-state-change-hooks conj #'persist-index-check-on-breaker-change!)++;;; ------------------------------------- semantic-search collectors ----------------------------------------++(def ^:private staleness-warn-seconds (* 60 60)) ; 1h -- indexer backlog under an hour reads healthy+(def ^:private staleness-critical-seconds (* 6 60 60)) ; 6h -- a backlog this old = the indexer is stuck+;; Absolute orphan counts. Repair clears garbage hourly, so a handful of in-flight orphans is normal; a large+;; count means repair isn't keeping up. Tunable (promotable to settings alongside the staleness thresholds).+(def ^:private garbage-warn-count 50)+(def ^:private garbage-critical-count 5000)+;; Repair runs hourly. Stop publishing its last snapshot after three missed runs, so an abandoned Quartz+;; owner cannot leave a plausible-looking value behind indefinitely.+(def ^:private repair-snapshot-max-age-seconds (* 3 60 60))++(defn- active-index*+ "The active semantic index and pgvector datasource, or nil when there is nothing applicable to measure."+ []+ (when (semantic.util/semantic-search-active?)+ (try+ (let [pgvector (semantic.env/get-pgvector-datasource!)+ state (semantic.index-metadata/get-active-index-state pgvector (semantic.env/get-index-metadata))]+ (when (and state+ (active-index-queryable? pgvector (-> state :index :table-name)))+ {:pgvector pgvector, :state state}))+ (catch InterruptedException e+ (throw e))+ (catch Exception e+ (log/debug e "semantic active-index lookup failed for a metric collector")+ nil))))++(def ^:private active-index+ "Cached [[active-index*]] so collectors and the repair reporter can share the lookup."+ (memoize/ttl active-index* :ttl/threshold (* 30 1000)))++(defn- scalar-row [pgvector sql]+ (jdbc/execute-one! pgvector sql {:builder-fn jdbc.rs/as-unqualified-lower-maps}))++(defn- fresh-repair-age?+ [age]+ (and (number? age)+ (<= 0.0 (double age) repair-snapshot-max-age-seconds)))++(defn- repair-snapshot-age+ "Age of `snapshot-at` in seconds, measured exclusively on the pgvector clock."+ [pgvector snapshot-at]+ (:repair_age+ (scalar-row pgvector+ ["SELECT EXTRACT(EPOCH FROM (clock_timestamp() - CAST(? AS timestamptz))) AS repair_age"+ snapshot-at])))++(defn- semantic-coverage []+ (when-let [{:keys [pgvector state]} (active-index)]+ (when-let [snapshot-at (-> state :metadata-row :repair_snapshot_at)]+ (let [table (semantic.util/quote-table (-> state :index :table-name))+ gate (semantic.util/quote-table (:gate-table-name (semantic.env/get-index-metadata)))+ ;; The repair age and both populations share one pgvector statement snapshot. Tombstones leave+ ;; the expected population immediately; stale index rows remain garbage's responsibility.+ {:keys [indexed expected repair_age]}+ (scalar-row pgvector+ [(format (str "WITH observed AS (SELECT clock_timestamp() AS at) "+ "SELECT count(*) FILTER (WHERE EXISTS "+ "(SELECT 1 FROM %s i "+ "WHERE i.model = g.model AND i.model_id = g.model_id)) AS indexed, "+ "count(*) AS expected, "+ "EXTRACT(EPOCH FROM ((SELECT at FROM observed) - "+ "CAST(? AS timestamptz))) AS repair_age "+ "FROM %s g WHERE g.document_hash IS NOT NULL")+ table gate)+ snapshot-at])]+ (when (fresh-repair-age? repair_age)+ (search.index-health/coverage-result (or indexed 0) (or expected 0)))))))++(defn- semantic-staleness []+ (when-let [{:keys [pgvector state]} (active-index)]+ (let [index-metadata (semantic.env/get-index-metadata)+ gate (semantic.util/quote-table (:gate-table-name index-metadata))+ {:keys [id indexer_last_seen indexer_last_seen_id]} (:metadata-row state)+ ;; Oldest gate change past the indexer watermark. Content-hash-based: the gate suppresses no-op+ ;; updated_at bumps, so this is real pending work, not spurious timestamp drift. Age is computed on+ ;; the pgvector clock to match gated_at (set via clock_timestamp).+ ;; Composite (gated_at, id) comparison, matching the gate poll's consumption order -- a+ ;; timestamp-only bound would hide pending rows that share the watermark timestamp. A nil watermark+ ;; (indexer never ran) reads everything as pending; a nil id treats same-timestamp rows as pending+ ;; ('' sorts before every real gate id).+ gate-row (scalar-row pgvector+ [(format "WITH observed AS (SELECT clock_timestamp() AS at)+ SELECT count(*) AS pending,+ EXTRACT(EPOCH FROM ((SELECT at FROM observed) - min(gated_at))) AS age,+ (SELECT at FROM observed) AS observed_at+ FROM %s+ WHERE (gated_at, id) > (COALESCE(?, '-infinity'::timestamptz), COALESCE(?, ''))"+ gate)+ indexer_last_seen indexer_last_seen_id])+ dlq-row (if (semantic.dlq/dlq-table-exists? pgvector index-metadata id)+ (let [table (semantic.util/quote-table+ (name (semantic.dlq/dlq-table-name-kw index-metadata id)))]+ (scalar-row pgvector+ [(format "SELECT count(*) AS pending,+ EXTRACT(EPOCH FROM (CAST(? AS timestamptz) - min(d.error_gated_at))) AS age+ FROM %s d+ JOIN %s g ON g.id = d.gate_id+ AND g.gated_at = d.error_gated_at"+ table gate)+ (:observed_at gate-row)]))+ {:pending 0, :age nil})+ gate-pending (or (:pending gate-row) 0)+ dlq-pending (or (:pending dlq-row) 0)+ ages (keep :age [gate-row dlq-row])+ age (when (seq ages) (apply max (map double ages)))+ detail (cond+ (and (pos? gate-pending) (pos? dlq-pending))+ (format "%d change(s) in the indexer backlog; %d failed change(s) awaiting DLQ retry."+ gate-pending dlq-pending)++ (pos? gate-pending)+ (format "%d change(s) in the indexer backlog." gate-pending)++ (pos? dlq-pending)+ (format "%d failed change(s) awaiting DLQ retry." dlq-pending))]+ (if (some #(neg? (double %)) ages)+ (search.index-health/degraded "Index staleness unavailable: database clock precedes a pending change.")+ (search.index-health/staleness-result+ age staleness-warn-seconds staleness-critical-seconds detail)))))++(defn- semantic-garbage+ ;; The repair job writes a time-bounded snapshot to shared index metadata. Every node therefore publishes+ ;; the same value even when Quartz moves the job between nodes.+ []+ (when-let [{:keys [pgvector state]} (active-index)]+ (let [{:keys [repair_orphan_count repair_snapshot_at]} (:metadata-row state)]+ (when (and (some? repair_orphan_count)+ repair_snapshot_at+ (fresh-repair-age? (repair-snapshot-age pgvector repair_snapshot_at)))+ (search.index-health/garbage-result+ repair_orphan_count garbage-warn-count garbage-critical-count)))))++(search.index-health/register-index-check! :semantic-search :coverage semantic-coverage)++(search.index-health/register-index-check! :semantic-search :staleness semantic-staleness)++(def ^:private semantic-garbage-measure+ "The semantic-search garbage descriptor. Holding it lets [[report-repair-metrics!]] refresh the metric+ when a repair pushes a new count instead of waiting for the next collector cycle."+ (search.index-health/register-index-check! :semantic-search :garbage semantic-garbage))++(defn- clear-active-index-cache! []+ (memoize/memo-clear! active-index))++(defn- store-repair-metrics!+ [index-id {:keys [orphan-count snapshot-at]}]+ (let [pgvector (semantic.env/get-pgvector-datasource!)+ index-metadata (semantic.env/get-index-metadata)+ index-id (or index-id+ (-> (semantic.index-metadata/get-active-index-state pgvector index-metadata)+ :metadata-row+ :id))+ metadata-table (-> index-metadata :metadata-table-name semantic.util/quote-table)]+ (when index-id+ (jdbc/execute-one!+ pgvector+ [(format (str "UPDATE %s SET "+ "repair_orphan_count = ?, "+ "repair_snapshot_at = ? "+ "WHERE id = ?")+ metadata-table)+ orphan-count snapshot-at index-id])+ (clear-active-index-cache!))))++(defn report-repair-metrics!+ "Publish the repair job's stale-orphan count through shared index metadata.+ `index-id` pins the snapshot to the index that repair measured; `:snapshot-at` is the time captured before+ reading the canonical document stream. Nil counts invalidate the measure. Reporting failures do not fail+ repair; interruption still propagates."+ ([counts]+ (report-repair-metrics! nil counts))+ ([index-id counts]+ (try+ (store-repair-metrics! index-id counts)+ (search.index-health/refresh-index-check! semantic-garbage-measure)+ (catch InterruptedException e+ (throw e))+ (catch Exception e+ (log/warn e "Failed to report semantic repair metrics")))))diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/index_metadata.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/index_metadata.cljindex 67aa335c3de..d68d7c3cf61 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/index_metadata.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/index_metadata.clj@@ -38,7 +38,9 @@[:indexer_last_seen :timestamp-with-time-zone :null][:indexer_last_seen_id :text :null][:indexer_last_seen_hash :text :null]- [:indexer_stalled_at :timestamp-with-time-zone :null]]+ [:indexer_stalled_at :timestamp-with-time-zone :null]+ [:repair_orphan_count :bigint :null]+ [:repair_snapshot_at :timestamp-with-time-zone :null]]:control[[:id :bigint [:primary-key]] ;; not auto-inc, only one row - still useful to ensure only one row when inserting.@@ -118,6 +120,17 @@(sql.helpers/with-columns schema)(sql/format :quoted true))))+(defn ensure-health-metric-columns!+ "Add the non-destructive health-metric columns to an existing metadata table."+ [pgvector {:keys [metadata-table-name]}]+ (when (semantic.util/table-exists? pgvector metadata-table-name)+ (jdbc/execute!+ pgvector+ [(format (str "ALTER TABLE %s "+ "ADD COLUMN IF NOT EXISTS repair_orphan_count bigint NULL, "+ "ADD COLUMN IF NOT EXISTS repair_snapshot_at timestamptz NULL")+ (semantic.util/quote-table metadata-table-name))])))+(comment(create-index-metadata-table-if-not-exists-sql default-index-metadata)(create-control-table-if-not-exists-sql default-index-metadata)@@ -144,7 +157,7 @@nil))(defn create-tables-if-not-exists!- "Creates the metadata and control tables if they do not already exist"+ "Create the metadata, control, and gate tables if needed."[pgvector index-metadata](let [{:keys [metadata-table-name control-table-name gate-table-name version]} index-metadata];; **note** we do not currently deal with version mismatches as there is only one version.diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/init.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/init.cljindex 07908499c6c..61d95f9e33f 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/init.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/init.clj@@ -2,6 +2,7 @@(:require[metabase-enterprise.semantic-search.embedding-health][metabase-enterprise.semantic-search.events]+ [metabase-enterprise.semantic-search.health][metabase-enterprise.semantic-search.settings][metabase-enterprise.semantic-search.task.index-cleanup][metabase-enterprise.semantic-search.task.index-repair]diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/repair.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/repair.cljindex da233f0d664..122071bd44a 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/repair.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/repair.clj@@ -37,31 +37,27 @@(log/debugf "Populated repair table with %d document records" (count repair-records)))))(defn find-lost-deletes- "Performs an anti-join to find documents that exist in the gate table- but are not in the repair table. These represent lost deletes."+ "Return gate documents absent from the repair table, representing lost deletes.++ Throws when the anti-join fails so the repair cannot publish a snapshot from incomplete evidence."[pgvector gate-table-name repair-table-name]- (try- (let [column semantic.util/column-keyword- anti-join-sql (-> (sql.helpers/select :model :model_id)- (sql.helpers/from (keyword gate-table-name))- (sql.helpers/where [:not [:exists- (-> (sql.helpers/select 1)- (sql.helpers/from (keyword repair-table-name))- (sql.helpers/where [:and- [:= (column repair-table-name "model")- (column gate-table-name "model")]- [:= (column repair-table-name "model_id")- (column gate-table-name "model_id")]]))]])- (sql/format :quoted true))- results (jdbc/execute! pgvector anti-join-sql {:builder-fn jdbc.rs/as-unqualified-lower-maps})]- (log/infof "Found %d documents in gate table that should be deleted" (count results))- results)- ;; TODO (Chris 2026-07-13) -- swallowing the error reads as "no lost deletes", so a transient anti-join- ;; failure silently skips tombstoning that run. Tolerated because the next repair pass retries; revisit- ;; if lost deletes start slipping through.- (catch Exception e- (log/errorf e "Error finding lost deletes between gate table %s and repair table %s"- gate-table-name repair-table-name))))+ (let [column semantic.util/column-keyword+ anti-join-sql (sql/format+ {:select [:model :model_id]+ :from [(keyword gate-table-name)]+ :where [:not+ [:exists+ {:select [1]+ :from [(keyword repair-table-name)]+ :where [:and+ [:= (column repair-table-name "model")+ (column gate-table-name "model")]+ [:= (column repair-table-name "model_id")+ (column gate-table-name "model_id")]]}]]}+ :quoted true)+ results (jdbc/execute! pgvector anti-join-sql {:builder-fn jdbc.rs/as-unqualified-lower-maps})]+ (log/infof "Found %d documents in gate table that should be deleted" (count results))+ results))(defn find-lost-deletes-by-model"Finds lost deletes and groups them by model for easier processing.@@ -73,6 +69,75 @@(group-by :model)(m/map-vals #(map :model_id %)))))+(defn count-stale-orphans+ "Count rows in the active index table whose `(model, model_id)` is absent from the repair table (the current+ candidate set) AND has no live or pending gate row -- garbage whose gated delete the indexer has already+ consumed yet whose index row still stands. `metadata-row` supplies the indexer's composite+ `(indexer_last_seen, indexer_last_seen_id)` consumption watermark; only tombstones at/behind it count, so+ in-flight deletes and current DLQ retries (the staleness metric's backlog) don't read as garbage.+ Callers run this BEFORE the current repair's own gate-deletes, for the same reason: a freshly found lost+ delete would otherwise push a garbage spike that stands until the next hourly repair.+ Returns nil if the count query fails: this feeds only the garbage health metric, so a metric-read blip+ must not fail the repair run whose real work already committed."+ [pgvector index-table-name gate-table-name repair-table-name dlq-table-name metadata-row]+ ;; Unlike find-lost-deletes' gate-based anti-join, measuring on the index table excludes retained+ ;; tombstones for rows the indexer already removed, which would keep the count inflated until tombstone+ ;; cleanup runs.+ (try+ (let [{:keys [indexer_last_seen indexer_last_seen_id]} metadata-row+ count-sql (sql/format+ {:select [[:%count.* :n]]+ :from [[(keyword index-table-name) :i]]+ :where [:and+ [:not+ [:exists+ {:select [1]+ :from [[(keyword repair-table-name) :r]]+ :where [:and+ [:= :r.model :i.model]+ [:= :r.model_id :i.model_id]]}]]+ ;; no live gate row (document_hash NULL or row gone) and no pending tombstone the+ ;; indexer hasn't consumed yet ((gated_at, id) past the watermark). A nil watermark+ ;; (indexer never ran) reads all tombstones as pending; a nil watermark id treats+ ;; same-timestamp tombstones as pending ('' sorts before every real gate id) --+ ;; conservative in both cases, no false spike.+ [:not+ [:exists+ {:select [1]+ :from [[(keyword gate-table-name) :g]]+ :where [:and+ [:= :g.model :i.model]+ [:= :g.model_id :i.model_id]+ [:or+ [:!= :g.document_hash nil]+ [:> [:composite :g.gated_at :g.id]+ [:composite+ [:coalesce [:lift indexer_last_seen]+ [:raw "'-infinity'::timestamptz"]]+ [:coalesce [:lift indexer_last_seen_id] ""]]]]]}]]+ ;; A failed delete may sit behind the watermark: stalled-mode advances it after+ ;; moving the failure to the DLQ. The current gate generation still owns that retry,+ ;; so it is pending work rather than garbage.+ [:not+ [:exists+ {:select [1]+ :from [[(keyword dlq-table-name) :d]]+ :join [[(keyword gate-table-name) :dg]+ [:and+ [:= :dg.id :d.gate_id]+ [:= :dg.gated_at :d.error_gated_at]]]+ :where [:and+ [:= :dg.model :i.model]+ [:= :dg.model_id :i.model_id]]}]]]}+ :quoted true)]+ (or (:n (jdbc/execute-one! pgvector count-sql {:builder-fn jdbc.rs/as-unqualified-lower-maps})) 0))+ (catch InterruptedException e+ (throw e))+ (catch Exception e+ (log/errorf e "Error counting stale orphans in index table %s against repair table %s"+ index-table-name repair-table-name)+ nil)))+(defn- create-repair-table!"Creates an empty temporary table for tracking documents during index repair."[pgvector repair-table-name]diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/task/index_repair.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/task/index_repair.cljindex e2618064baf..f699e1e34d0 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/task/index_repair.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/task/index_repair.clj@@ -5,6 +5,7 @@[clojurewerkz.quartzite.schedule.cron :as cron][clojurewerkz.quartzite.triggers :as triggers][metabase-enterprise.semantic-search.core :as semantic-search.core]+ [metabase-enterprise.semantic-search.health :as semantic.health][metabase-enterprise.semantic-search.util :as semantic.u][metabase.lib-be.core :as lib-be][metabase.search.ingestion :as search.ingestion]@@ -18,20 +19,30 @@(def ^:private repair-job-key (jobs/key "metabase.task.semantic-index-repair.job"))(def ^:private repair-trigger-key (triggers/key "metabase.task.semantic-index-repair.trigger"))+(defn- repair-index! []+ (try+ (log/info "Starting semantic search index repair")+ ;; Keep the cache bound while the document eduction is realized inside repair-index!.+ (lib-be/with-metadata-provider-cache+ (let [{:keys [index-id orphans snapshot-at]}+ (semantic-search.core/repair-index! (search.ingestion/searchable-documents))]+ (semantic.health/report-repair-metrics! index-id {:orphan-count orphans+ :snapshot-at snapshot-at})))+ (log/info "Completed semantic search index repair")+ (catch InterruptedException e+ (semantic.health/report-repair-metrics! nil)+ (throw e))+ (catch Exception e+ ;; Invalidate the shared snapshot when the producing job fails.+ (semantic.health/report-repair-metrics! nil)+ (log/error e "Failed to complete semantic search index repair"))))+(task/defjob ^{DisallowConcurrentExecution true:doc "Runs repair-index! to maintain semantic search consistency"}SemanticIndexRepair [_ctx](when (semantic.u/semantic-search-active?)(log/with-context {:quartz-job-type 'SemanticIndexRepair}- (try- (log/info "Starting semantic search index repair")- ;; The document eduction is realized inside repair-index!, so the metadata-provider cache must be- ;; bound around the whole call for spec :fn attrs that touch metadata to benefit from it.- (lib-be/with-metadata-provider-cache- (semantic-search.core/repair-index! (search.ingestion/searchable-documents)))- (log/info "Completed semantic search index repair")- (catch Exception e- (log/error e "Failed to complete semantic search index repair"))))))+ (repair-index!))))(defmethod task/init! ::SemanticIndexRepair [_](when (semantic.u/semantic-search-configured?)diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/util.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/util.cljindex 839790af25b..37e19af8307 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/util.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/util.clj@@ -25,6 +25,15 @@[table-name](second (qualified-table-parts table-name)))+(defn quote-table+ "Quote a possibly schema-qualified `table-name` for raw SQL, quoting schema and table separately so it+ renders as \"schema\".\"table\" rather than one identifier with a literal dot."+ [table-name]+ (let [[schema table] (qualified-table-parts table-name)]+ (if schema+ (str (quote-ident schema) "." (quote-ident table))+ (quote-ident table))))+(defn column-keyword"A `table.column` reference as a dotted-name keyword, not a namespaced one.HoneySQL quotes a keyword's namespace as one identifier, so a schema-qualified table there renders as a@@ -69,6 +78,26 @@{:builder-fn jdbc.rs/as-unqualified-lower-maps})(:index_exists false))))+(defn index-ready?+ "Whether `index-name` is present and marked ready and valid by PostgreSQL. An in-progress or abandoned+ concurrent build is not ready. A schema-qualified name matches only within its schema."+ [pgvector index-name]+ (let [[schema index] (qualified-table-parts index-name)]+ (-> (jdbc/execute-one! pgvector+ (if schema+ [(str "SELECT exists (SELECT 1 FROM pg_class i "+ "JOIN pg_namespace n ON n.oid = i.relnamespace "+ "JOIN pg_index x ON x.indexrelid = i.oid "+ "WHERE n.nspname = ? AND i.relname = ? "+ "AND x.indisready AND x.indisvalid) AS index_ready")+ schema index]+ [(str "SELECT exists (SELECT 1 FROM pg_class i "+ "JOIN pg_index x ON x.indexrelid = i.oid "+ "WHERE i.relname = ? AND x.indisready AND x.indisvalid) AS index_ready")+ index])+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})+ (:index_ready false))))+(defn semantic-search-configured?"Whether to schedule the semantic-search Quartz jobs at startup.True when the `:semantic-search` feature is present and a pgvector store might exist: a dedicateddiff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/core_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/core_test.cljindex f6c64514d7a..b189edbe209 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/core_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/core_test.clj@@ -4,7 +4,9 @@[metabase-enterprise.semantic-search.core :as semantic.core][metabase-enterprise.semantic-search.embedding :as semantic.embedding][metabase-enterprise.semantic-search.env :as semantic.env]+ [metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata][metabase-enterprise.semantic-search.pgvector-api :as semantic.pgvector-api]+ [metabase-enterprise.semantic-search.repair :as semantic.repair][metabase-enterprise.semantic-search.test-util :as semantic.tu][metabase-enterprise.semantic-search.util :as semantic.util][metabase.analytics-interface.core :as analytics]@@ -16,6 +18,8 @@[metabase.search.settings :as search.settings][metabase.test :as mt]))+(set! *warn-on-reflection* true)+(use-fixtures :once #'semantic.tu/once-fixture)(deftest supported?-requires-an-embedder-test@@ -30,6 +34,32 @@(constantly semantic.tu/mock-embedding-model)](is (true? (semantic.core/supported?))))))))+(deftest ^:sequential repair-snapshot-precedes-canonical-document-read-test+ (let [events (atom [])+ active-state (atom [nil {:metadata-row {:id 17}}])+ documents (map (fn [document]+ (swap! events conj :document-read)+ document)+ [{}])]+ (mt/with-premium-features #{:semantic-search}+ (mt/with-dynamic-fn-redefs+ [semantic.core/capture-repair-snapshot-at (fn [_]+ (swap! events conj :snapshot)+ ::snapshot-at)+ semantic.env/get-pgvector-datasource! (constantly ::pgvector)+ semantic.env/get-index-metadata (constantly ::index-metadata)+ semantic.env/get-configured-embedding-model (constantly ::embedding-model)+ semantic.index-metadata/get-active-index-state (fn [& _]+ (let [state (first @active-state)]+ (swap! active-state subvec 1)+ state))+ semantic.pgvector-api/init-semantic-search! (fn [& _])+ semantic.repair/with-repair-table! (fn [_ _ f] (f "repair"))+ semantic.pgvector-api/gate-updates! (fn [_ _ docs & _] (dorun docs))]+ (is (= {:index-id 17, :orphans 0, :snapshot-at ::snapshot-at}+ (semantic.core/repair-index! documents))))+ (is (= [:snapshot :document-read] @events)))))+(deftest fallback-engine-available-with-semantic-test(mt/with-premium-features #{:semantic-search}(mt/with-temporary-setting-values [search.settings/search-engine "semantic"]diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_circuit_breaker_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_circuit_breaker_test.cljindex 9c09f1afc6e..5aa517beb51 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_circuit_breaker_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_circuit_breaker_test.clj@@ -3,8 +3,12 @@[clj-http.client :as http][clojure.test :refer :all][diehard.circuit-breaker :as dh.cb]+ ;; loaded for side effects: the health namespaces register the breaker state-change hooks that+ ;; state-change-persists-affected-checks-test exercises+ [metabase-enterprise.entity-retrieval.health][metabase-enterprise.semantic-search.embedding :as semantic.embedding][metabase-enterprise.semantic-search.embedding-health :as embedding-health]+ [metabase-enterprise.semantic-search.health][metabase-enterprise.semantic-search.models.token-tracking :as token-tracking][metabase-enterprise.semantic-search.settings :as semantic.settings][metabase.analytics-interface.core :as analytics]@@ -303,6 +307,19 @@(#'embedding-health/request-recovery-on-open! :open)(is (= 1 @requests)))))+(deftest ^:sequential state-change-persists-affected-checks-test+ (testing "a breaker state transition runs the registered hooks, persisting both embedder-dependent health+ checks immediately"+ (let [persisted (atom [])+ completed (CountDownLatch. 2)]+ (mt/with-dynamic-fn-redefs+ [health-inspector/run-and-save-check! (fn [check-name]+ (swap! persisted conj check-name)+ (.countDown completed))]+ (#'semantic.embedding/on-embedder-circuit-state-change! :open)+ (is (.await completed 5 TimeUnit/SECONDS) "state-change hooks did not finish")+ (is (= #{:semantic-search-index :nlq-retrieval} (set @persisted)))))))+(deftest ^:sequential state-changes-run-serially-in-arrival-order-test(testing "transitions queue through one agent: a later transition's hooks can't overtake an earliertransition whose hook is still blocked (e.g. on a probe against a down service), so a slowdiff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/health_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/health_test.cljnew file mode 100644index 00000000000..6eb212a8851--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/health_test.clj@@ -0,0 +1,486 @@+(ns metabase-enterprise.semantic-search.health-test+ (:require+ [clojure.core.memoize :as memoize]+ [clojure.test :refer :all]+ [java-time.api :as t]+ [metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource]+ [metabase-enterprise.semantic-search.dlq :as semantic.dlq]+ [metabase-enterprise.semantic-search.embedding :as semantic.embedding]+ [metabase-enterprise.semantic-search.embedding-health :as embedding-health]+ [metabase-enterprise.semantic-search.env :as semantic.env]+ [metabase-enterprise.semantic-search.health :as semantic.health]+ [metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata]+ [metabase-enterprise.semantic-search.settings :as semantic.settings]+ [metabase-enterprise.semantic-search.util :as semantic.util]+ [metabase.analytics-interface.core :as analytics]+ [metabase.health-inspector.core :as health-inspector]+ [metabase.search.index-health :as search.index-health]+ [metabase.test :as mt]+ [next.jdbc :as jdbc]+ [next.jdbc.result-set :as jdbc.rs])+ (:import+ (java.util.concurrent CountDownLatch TimeUnit)))++(set! *warn-on-reflection* true)++(def ^:private embedding-model+ {:provider "ai-service", :model-name "test-model", :vector-dimensions 3})++(def ^:private active-state+ {:index {:table-name "index_test_table", :embedding-model embedding-model}+ :metadata-row {:id 1+ :indexer_stalled_at nil+ :repair_orphan_count nil+ :repair_snapshot_at nil}})++(defn- do-with-enabled!+ "Run `thunk` with the semantic engine active and the pgvector accessors stubbed to harmless values, so+ each test only has to override the signal it cares about."+ [thunk]+ (mt/with-dynamic-fn-redefs+ [semantic.util/semantic-search-active? (constantly true)+ semantic.env/get-pgvector-datasource! (constantly ::pgvector)+ semantic.env/get-index-metadata (constantly ::index-metadata)+ semantic.env/get-configured-embedding-model (constantly embedding-model)+ semantic.settings/semantic-search-vector-strategy (constantly :brute-force)+ embedding-health/request-circuit-recovery! (constantly nil)]+ (thunk)))++(deftest not-enabled-is-omitted-test+ (testing "an inactive engine (unlicensed, unconfigured, or another engine selected) -> nil (omitted, not a+ misleading 'No active semantic search index' incident)"+ (mt/with-dynamic-fn-redefs [semantic.util/semantic-search-active? (constantly false)]+ (is (nil? (semantic.health/index-health-check))))))++(deftest pgvector-unreachable-test+ (testing "a pgvector error while resolving the active index reports as degraded, not as no-index"+ (do-with-enabled!+ (fn []+ (mt/with-dynamic-fn-redefs+ [semantic.index-metadata/get-active-index-state (fn [& _] (throw (ex-info "connection refused" {})))]+ (is (=? {:health 0 :message #".*pgvector store unreachable: connection refused.*"}+ (semantic.health/index-health-check))))))))++(deftest health-probes-propagate-interruption-test+ (testing "active-index lookup interruption propagates"+ (do-with-enabled!+ (fn []+ (mt/with-dynamic-fn-redefs+ [semantic.index-metadata/get-active-index-state (fn [& _] (throw (InterruptedException.)))]+ (is (thrown? InterruptedException (semantic.health/index-health-check)))))))+ (testing "queryability probe interruption propagates"+ (mt/with-dynamic-fn-redefs [jdbc/execute-one! (fn [& _] (throw (InterruptedException.)))]+ (is (thrown? InterruptedException+ (#'semantic.health/active-index-queryable? ::pgvector "index_table"))))))++(deftest metric-active-index-requires-queryable-table-test+ (do-with-enabled!+ (fn []+ (mt/with-dynamic-fn-redefs+ [semantic.index-metadata/get-active-index-state (constantly active-state)+ semantic.health/active-index-queryable? (constantly false)]+ (is (nil? (#'semantic.health/active-index*)))))))++(deftest no-active-index-test+ (testing "enabled but no active index -> degraded"+ (do-with-enabled!+ (fn []+ (mt/with-dynamic-fn-redefs+ [semantic.index-metadata/get-active-index-state (constantly nil)]+ (is (=? {:health 0 :message #"No active semantic search index\."}+ (semantic.health/index-health-check))))))))++(deftest healthy-test+ (testing "active + queryable + un-stalled + embedder reachable -> healthy 100"+ (do-with-enabled!+ (fn []+ (mt/with-dynamic-fn-redefs+ [semantic.index-metadata/get-active-index-state (constantly active-state)+ semantic.health/active-index-queryable? (constantly true)+ semantic.embedding/embedder-circuit-untrusted? (constantly false)+ embedding-health/embedding-service-reachable? (constantly {:reachable? true :error nil})]+ (is (=? {:health 100 :message #"Semantic search index active.*serving\."}+ (semantic.health/index-health-check))))))))++(deftest degraded-conditions-test+ (do-with-enabled!+ (fn []+ (testing "a stalled indexer degrades and names the stall"+ (mt/with-dynamic-fn-redefs+ [semantic.index-metadata/get-active-index-state+ (constantly (assoc-in active-state [:metadata-row :indexer_stalled_at] "2026-07-07"))+ semantic.health/active-index-queryable? (constantly true)+ semantic.embedding/embedder-circuit-untrusted? (constantly false)+ embedding-health/embedding-service-reachable? (constantly {:reachable? true :error nil})]+ (is (=? {:health 0 :message #".*indexer stalled since 2026-07-07.*"}+ (semantic.health/index-health-check)))))+ (testing "an unqueryable active table degrades"+ (mt/with-dynamic-fn-redefs+ [semantic.index-metadata/get-active-index-state (constantly active-state)+ semantic.health/active-index-queryable? (constantly false)+ semantic.embedding/embedder-circuit-untrusted? (constantly false)+ embedding-health/embedding-service-reachable? (constantly {:reachable? true :error nil})]+ (is (=? {:health 0 :message #".*active index table not queryable.*"}+ (semantic.health/index-health-check)))))+ (testing "an index-backed strategy without its HNSW index degrades"+ (mt/with-dynamic-fn-redefs+ [semantic.index-metadata/get-active-index-state (constantly active-state)+ semantic.health/active-index-queryable? (constantly true)+ semantic.settings/semantic-search-vector-strategy (constantly :hnsw)+ semantic.util/index-ready? (constantly false)+ semantic.embedding/embedder-circuit-untrusted? (constantly false)+ embedding-health/embedding-service-reachable? (constantly {:reachable? true :error nil})]+ (is (=? {:health 0 :message #".*required HNSW index is missing.*"}+ (semantic.health/index-health-check)))))+ (testing "an active index for a different embedding model degrades without probing the wrong model"+ (mt/with-dynamic-fn-redefs+ [semantic.index-metadata/get-active-index-state+ (constantly (assoc-in active-state [:index :embedding-model :model-name] "old-model"))+ semantic.health/active-index-queryable? (constantly true)+ embedding-health/embedding-problem #(throw (ex-info "must not probe configured model" {}))]+ (is (=? {:health 0 :message #".*active index embedding model does not match configured model.*"}+ (semantic.health/index-health-check)))))+ (testing "a non-closed breaker (open or half-open) still probes (so a recovered-but-idle embedder is+ detectable) and degrades identically in both states"+ (let [probed? (atom false)]+ (mt/with-dynamic-fn-redefs+ [semantic.index-metadata/get-active-index-state (constantly active-state)+ semantic.health/active-index-queryable? (constantly true)+ semantic.embedding/embedder-circuit-untrusted? (constantly true)+ embedding-health/embedding-service-reachable? (fn [] (reset! probed? true) {:reachable? true})]+ (is (=? {:health 0 :message #".*circuit open \(probe reachable.*"}+ (semantic.health/index-health-check)))+ (is (true? @probed?) "an open circuit still probes so recovery is detectable"))))+ (testing "an open circuit with an unreachable probe reads the same as unreachable with a closed one --+ state-independent wording, so a flapping breaker's re-persisted rows dedup instead of flooding"+ (mt/with-dynamic-fn-redefs+ [semantic.index-metadata/get-active-index-state (constantly active-state)+ semantic.health/active-index-queryable? (constantly true)+ semantic.embedding/embedder-circuit-untrusted? (constantly true)+ embedding-health/embedding-service-reachable? (constantly {:reachable? false :error "boom"})]+ (is (=? {:health 0 :message #".*embedding service unreachable: boom.*"}+ (semantic.health/index-health-check)))))+ (testing "an unreachable embedder degrades and names the error"+ (mt/with-dynamic-fn-redefs+ [semantic.index-metadata/get-active-index-state (constantly active-state)+ semantic.health/active-index-queryable? (constantly true)+ semantic.embedding/embedder-circuit-untrusted? (constantly false)+ embedding-health/embedding-service-reachable? (constantly {:reachable? false :error "boom"})]+ (is (=? {:health 0 :message #".*embedding service unreachable: boom.*"}+ (semantic.health/index-health-check))))))))++;; Tests the raw probe (`embedding-service-reachable?` is TTL-memoized, so calling it across cases would+;; return a stale cached result).+;; ^:sequential: redefs the process-global `get-configured-model` / `get-embedding` via with-redefs, so it+;; must not run concurrently with other tests (e.g. entity-retrieval-available? reads get-configured-model).+(deftest ^:sequential probe-embedding-service-test+ (testing "a successful embed reads as reachable; the probe bypasses the breaker and silences snowplow"+ (let [seen (atom nil)]+ ;; get-embedding is a multimethod; with-dynamic-fn-redefs can't patch those, so with-redefs is required.+ (with-redefs [semantic.embedding/get-configured-model (constantly {:model-name "m"})+ semantic.embedding/get-embedding+ (fn [_model _text opts]+ (reset! seen (assoc opts :bypass semantic.embedding/*bypass-circuit-breaker*))+ [0.1 0.2])]+ (is (=? {:reachable? true :error nil} (#'embedding-health/probe-embedding-service)))+ (is (=? {:bypass true :snowplow? false :record-tokens? false} @seen)+ "the probe runs with the breaker bypassed, snowplow silenced, and tokens unrecorded"))))+ (testing "an embedding failure reads as unreachable and captures the message"+ (with-redefs [semantic.embedding/get-configured-model (constantly {:model-name "m"})+ semantic.embedding/get-embedding (fn [& _] (throw (ex-info "connection refused" {})))]+ (is (=? {:reachable? false :error "connection refused"}+ (#'embedding-health/probe-embedding-service))))))++(deftest ^:sequential breaker-transition-probe-cache-test+ (testing "open and half-open transitions reuse the probe; recovery clears it"+ (let [probes (atom 0)]+ ;; get-embedding is a multimethod; with-dynamic-fn-redefs can't patch those, so with-redefs is required.+ (with-redefs [semantic.embedding/get-configured-model (constantly {:model-name "m"})+ semantic.embedding/get-embedding (fn [& _] (swap! probes inc) [0.1])]+ (mt/with-dynamic-fn-redefs [health-inspector/run-and-save-check! (constantly nil)]+ (memoize/memo-clear! @#'embedding-health/embedding-service-reachable?*)+ (embedding-health/embedding-service-reachable?)+ (is (= 1 @probes))+ (#'embedding-health/clear-probe-cache-on-recovery! :open)+ (#'embedding-health/clear-probe-cache-on-recovery! :half-open)+ (embedding-health/embedding-service-reachable?)+ (is (= 1 @probes) "an open<->half-open flap rides the cached probe")+ (#'embedding-health/clear-probe-cache-on-recovery! :closed)+ (embedding-health/embedding-service-reachable?)+ (is (= 2 @probes) "recovery busts the cache so the persisted row can't be a stale 'unreachable'")+ (memoize/memo-clear! @#'embedding-health/embedding-service-reachable?*))))))++(deftest ^:sequential circuit-recovery-probe-test+ (testing "recovery runs off-thread and supplies enough successful trials to close an untrusted circuit"+ (let [calls (atom 0)+ completed (CountDownLatch. 1)+ state @#'embedding-health/recovery-state+ before @state]+ (try+ (reset! state {:running? false, :last-start-ns nil})+ (with-redefs [semantic.embedding/get-configured-model (constantly {:model-name "m"})+ semantic.embedding/get-embedding+ (fn [& _]+ (when (= 2 (swap! calls inc))+ (.countDown completed))+ [0.1])]+ (mt/with-dynamic-fn-redefs+ [semantic.embedding/embedder-circuit-untrusted? #(< @calls 2)+ embedding-health/schedule-recovery! future-call]+ (is (nil? (embedding-health/request-circuit-recovery!)))+ (is (.await completed 5 TimeUnit/SECONDS) "recovery trials did not finish")+ (is (= 2 @calls))))+ (finally+ (reset! state before))))))++(deftest semantic-coverage-tracks-live-gate-after-repair-snapshot-test+ (when semantic.db.datasource/db-url+ (let [pgvector (semantic.db.datasource/ensure-initialized-data-source!)+ suffix (System/nanoTime)+ index-t (str "coverage_index_" suffix)+ gate-t (str "coverage_gate_" suffix)+ snapshot-at (:snapshot_at+ (jdbc/execute-one! pgvector ["SELECT clock_timestamp() AS snapshot_at"]+ {:builder-fn jdbc.rs/as-unqualified-lower-maps}))+ exec! #(jdbc/execute! pgvector [%])+ coverage (fn []+ (mt/with-dynamic-fn-redefs+ [semantic.health/active-index+ (constantly {:pgvector pgvector+ :state (-> active-state+ (assoc-in [:index :table-name] index-t)+ (assoc-in [:metadata-row :repair_orphan_count] 0)+ (assoc-in [:metadata-row :repair_snapshot_at]+ snapshot-at))})+ semantic.env/get-index-metadata (constantly {:gate-table-name gate-t})]+ (#'semantic.health/semantic-coverage)))]+ (try+ (exec! (format "CREATE TABLE \"%s\" (model text, model_id text)" index-t))+ (exec! (format "CREATE TABLE \"%s\" (model text, model_id text, document_hash text)" gate-t))+ (exec! (format "INSERT INTO \"%s\" VALUES ('card','a','ha'),('card','b','hb')" gate-t))+ (exec! (format "INSERT INTO \"%s\" VALUES ('card','a')" index-t))+ (testing "coverage starts from the live gate and active index"+ (is (=? {:value 0.5, :health 50, :message "1 of 2 expected items indexed (50%)."}+ (coverage))))+ (testing "a post-repair addition lowers coverage until it is indexed"+ (exec! (format "INSERT INTO \"%s\" VALUES ('card','c','hc')" gate-t))+ (is (=? {:health 33, :message "1 of 3 expected items indexed (33%)."} (coverage)))+ (exec! (format "INSERT INTO \"%s\" VALUES ('card','c')" index-t))+ (is (=? {:health 67, :message "2 of 3 expected items indexed (67%)."} (coverage))))+ (testing "a tombstone leaves coverage immediately even while its index row lingers"+ (exec! (format "UPDATE \"%s\" SET document_hash = NULL WHERE model_id = 'a'" gate-t))+ (is (=? {:value 0.5, :health 50, :message "1 of 2 expected items indexed (50%)."}+ (coverage))))+ (testing "an orphan cannot inflate the numerator"+ (exec! (format "INSERT INTO \"%s\" VALUES ('card','orphan')" index-t))+ (is (=? {:value 0.5, :health 50, :message "1 of 2 expected items indexed (50%)."}+ (coverage))))+ (finally+ (exec! (format "DROP TABLE IF EXISTS \"%s\"" index-t))+ (exec! (format "DROP TABLE IF EXISTS \"%s\"" gate-t)))))))++(deftest semantic-coverage-requires-a-fresh-database-aged-repair-test+ (testing "no successful repair snapshot -> coverage is unknown without querying"+ (mt/with-dynamic-fn-redefs+ [semantic.health/active-index (constantly {:pgvector ::pgvector, :state active-state})+ semantic.health/scalar-row #(throw (ex-info "must not query" {}))]+ (is (nil? (#'semantic.health/semantic-coverage)))))+ (testing "future-skewed and expired snapshots are unknown, never healthy"+ (doseq [age [-1 10801]]+ (let [queries (atom [])]+ (mt/with-dynamic-fn-redefs+ [semantic.health/active-index+ (constantly {:pgvector ::pgvector+ :state (assoc-in active-state [:metadata-row :repair_snapshot_at] ::snapshot-at)})+ semantic.env/get-index-metadata (constantly {:gate-table-name "index_gate"})+ semantic.health/scalar-row (fn [_ sql]+ (swap! queries conj sql)+ {:indexed 1, :expected 1, :repair_age age})]+ (is (nil? (#'semantic.health/semantic-coverage))))+ (is (= ::snapshot-at (-> @queries first second))+ "the pgvector query receives the persisted database timestamp")+ (is (re-find #"clock_timestamp" (-> @queries first first))+ "repair age and coverage populations share one database statement")))))++(deftest semantic-staleness-composite-watermark-test+ ;; Hermetic: an ad-hoc gate table stands in, exercising the composite (gated_at, id) watermark SQL.+ (when semantic.db.datasource/db-url+ (let [pgvector (semantic.db.datasource/ensure-initialized-data-source!)+ gate-t (str "staleness_gate_" (System/nanoTime))+ wm-ts (t/offset-date-time "2026-01-01T12:00:00Z")+ staleness (fn [metadata-row]+ (mt/with-dynamic-fn-redefs+ [semantic.health/active-index (constantly {:pgvector pgvector+ :state {:metadata-row metadata-row}})+ semantic.env/get-index-metadata (constantly {:gate-table-name gate-t+ :index-table-qualifier "%s"})]+ (#'semantic.health/semantic-staleness)))+ backlog (fn [result] (or (some->> (:message result) (re-find #"(\d+) change\(s\)") second parse-long) 0))]+ (try+ (jdbc/execute! pgvector [(format "CREATE TABLE \"%s\" (id text, gated_at timestamptz)" gate-t)])+ (jdbc/execute! pgvector [(format (str "INSERT INTO \"%s\" VALUES "+ "('a', timestamptz '2026-01-01 11:00:00+00'), "+ "('b', timestamptz '2026-01-01 12:00:00+00'), "+ "('c', timestamptz '2026-01-01 12:00:00+00')")+ gate-t)])+ (testing "rows at the watermark timestamp with a later id are pending -- a timestamp-only bound+ would hide them and read a stalled indexer as current"+ (is (= 1 (backlog (staleness {:id 7, :indexer_last_seen wm-ts :indexer_last_seen_id "b"})))))+ (testing "a nil watermark (indexer never ran) reads everything as pending"+ (is (= 3 (backlog (staleness {:id 7, :indexer_last_seen nil :indexer_last_seen_id nil})))))+ (testing "a timestamp-only watermark (no id) treats same-timestamp rows as pending"+ (is (= 2 (backlog (staleness {:id 7, :indexer_last_seen wm-ts :indexer_last_seen_id nil})))))+ (testing "a watermark past every row reads current"+ (is (=? {:value 0, :health 100, :message #"Index current\."}+ (staleness {:id 7+ :indexer_last_seen (t/plus wm-ts (t/hours 1))+ :indexer_last_seen_id ""}))))+ (finally+ (jdbc/execute! pgvector [(format "DROP TABLE IF EXISTS \"%s\"" gate-t)]))))))++(deftest semantic-staleness-includes-active-dlq-test+ (let [rows (atom [{:pending 0, :age nil, :observed_at ::observed-at}+ {:pending 1, :age 25200}])+ queries (atom [])]+ (mt/with-dynamic-fn-redefs+ [semantic.health/active-index+ (constantly {:pgvector ::pgvector+ :state {:metadata-row {:id 7, :indexer_last_seen nil, :indexer_last_seen_id nil}}})+ semantic.env/get-index-metadata (constantly {:gate-table-name "index_gate"+ :index-table-qualifier "%s"})+ semantic.dlq/dlq-table-exists? (constantly true)+ semantic.health/scalar-row (fn [_ sql]+ (swap! queries conj sql)+ (let [row (first @rows)]+ (swap! rows subvec 1)+ row))]+ (is (=? {:value 25200+ :health 0+ :message #".*1 failed change\(s\) awaiting DLQ retry.*"}+ (#'semantic.health/semantic-staleness)))+ (is (re-find #"JOIN \"index_gate\" g ON g.id = d.gate_id\s+AND g.gated_at = d.error_gated_at"+ (-> @queries second first))+ "obsolete DLQ generations are excluded by matching the current gate timestamp")+ (is (= ::observed-at (-> @queries second second))+ "both backlog ages use the database time captured by the first query"))))++(deftest semantic-staleness-rejects-negative-database-age-test+ (mt/with-dynamic-fn-redefs+ [semantic.health/active-index+ (constantly {:pgvector ::pgvector+ :state {:metadata-row {:id 7, :indexer_last_seen nil, :indexer_last_seen_id nil}}})+ semantic.env/get-index-metadata (constantly {:gate-table-name "index_gate"+ :index-table-qualifier "%s"})+ semantic.dlq/dlq-table-exists? (constantly false)+ semantic.health/scalar-row (constantly {:pending 1, :age -1, :observed_at ::observed-at})]+ (is (=? {:health 0+ :message "Index staleness unavailable: database clock precedes a pending change."}+ (#'semantic.health/semantic-staleness)))))++(deftest ^:sequential report-repair-metrics!-test+ (mt/with-dynamic-fn-redefs [health-inspector/enabled? (constantly false)]+ (testing "a stored snapshot refreshes the garbage gauge immediately"+ (let [calls (atom [])+ shared (atom nil)]+ (mt/with-dynamic-fn-redefs+ [semantic.health/store-repair-metrics! (fn [_index-id counts] (reset! shared counts))+ semantic.health/active-index+ #(let [{:keys [orphan-count]} @shared]+ {:pgvector :x+ :state (update active-state :metadata-row assoc+ :repair_orphan_count orphan-count+ :repair_snapshot_at (t/offset-date-time))})+ semantic.health/repair-snapshot-age (constantly 0)+ analytics/set-gauge! (fn [& args] (swap! calls conj (vec args)))]+ (semantic.health/report-repair-metrics! {:orphan-count 3}))+ (is (= [[:metabase-search/index-garbage-count {:index "semantic-search"} 3]] @calls))))+ (testing "with no active index, the measure reads N/A instead of serving an old shared count"+ (let [calls (atom [])]+ (mt/with-dynamic-fn-redefs [semantic.health/store-repair-metrics! (constantly nil)+ semantic.health/active-index (constantly nil)+ analytics/set-gauge! (fn [& args] (swap! calls conj (vec args)))]+ (semantic.health/report-repair-metrics! {:orphan-count 3})+ (semantic.health/report-repair-metrics! nil))+ (is (every? (fn [[_gauge _labels value]] (and (double? value) (Double/isNaN ^double value))) @calls)+ "no real value reaches the gauge while there's no active index")))+ (testing "an ordinary metric-sink error does not fail the repair job"+ (mt/with-dynamic-fn-redefs [semantic.health/store-repair-metrics! (constantly nil)+ semantic.health/active-index+ (constantly {:pgvector :x+ :state (update active-state :metadata-row assoc+ :repair_orphan_count 3+ :repair_snapshot_at (t/offset-date-time))})+ semantic.health/repair-snapshot-age (constantly 0)+ analytics/set-gauge! (fn [& _] (throw (ex-info "boom" {})))]+ (is (nil? (semantic.health/report-repair-metrics! {:orphan-count 3})))))))++(deftest invalid-repair-snapshot-age-is-omitted-test+ (doseq [age [-1 10801]]+ (mt/with-dynamic-fn-redefs+ [semantic.health/active-index+ (constantly {:pgvector :x+ :state (update active-state :metadata-row assoc+ :repair_orphan_count 3+ :repair_snapshot_at ::snapshot-at)})+ semantic.health/repair-snapshot-age (constantly age)]+ (is (nil? (#'semantic.health/semantic-garbage))))))++(deftest store-repair-metrics!-writes-exact-index-metadata-test+ (let [calls (atom [])+ snapshot-at (t/offset-date-time)]+ (mt/with-dynamic-fn-redefs+ [semantic.env/get-pgvector-datasource! (constantly ::pgvector)+ semantic.env/get-index-metadata (constantly {:metadata-table-name "index_metadata"})+ semantic.index-metadata/get-active-index-state #(throw (ex-info "explicit index id must win" {}))+ semantic.health/clear-active-index-cache! (constantly nil)+ jdbc/execute-one! (fn [& args] (swap! calls conj args))]+ (#'semantic.health/store-repair-metrics! 17 {:orphan-count 7, :snapshot-at snapshot-at}))+ (is (= [7 snapshot-at 17] (-> @calls first second rest vec)))+ (is (re-find #"repair_orphan_count" (-> @calls first second first)))))++(deftest store-repair-metrics!-resolves-current-index-without-the-read-cache-test+ (let [calls (atom [])+ snapshot-at (t/offset-date-time)]+ (mt/with-dynamic-fn-redefs+ [semantic.env/get-pgvector-datasource! (constantly ::pgvector)+ semantic.env/get-index-metadata (constantly {:metadata-table-name "index_metadata"})+ semantic.index-metadata/get-active-index-state (constantly active-state)+ semantic.health/active-index (constantly nil)+ semantic.health/clear-active-index-cache! (constantly nil)+ jdbc/execute-one! (fn [& args] (swap! calls conj args))]+ (#'semantic.health/store-repair-metrics! nil {:orphan-count 7, :snapshot-at snapshot-at}))+ (is (= [7 snapshot-at 1] (-> @calls first second rest vec)))))++(deftest report-repair-metrics!-propagates-interruption-test+ (mt/with-dynamic-fn-redefs+ [semantic.health/store-repair-metrics! (fn [& _] (throw (InterruptedException.)))]+ (is (thrown? InterruptedException+ (semantic.health/report-repair-metrics! {:orphan-count 3})))))++(deftest ^:sequential refresh-clears-garbage-when-inactive-test+ (testing "refresh NaNs a previously-emitted semantic garbage series when there's no active index, since no+ repair push will clear it (the collector reads N/A)"+ (let [calls (atom [])]+ (mt/with-dynamic-fn-redefs [analytics/set-gauge! (fn [& args] (swap! calls conj (vec args)))]+ (mt/with-dynamic-fn-redefs [health-inspector/enabled? (constantly false)]+ ;; make the series live: a repair push while the feature was on+ (mt/with-dynamic-fn-redefs+ [semantic.health/store-repair-metrics! (constantly nil)+ semantic.health/active-index+ (constantly {:pgvector :x+ :state (update active-state :metadata-row assoc+ :repair_orphan_count 3+ :repair_snapshot_at (t/offset-date-time))})]+ (mt/with-dynamic-fn-redefs+ [semantic.health/repair-snapshot-age (constantly 0)]+ (semantic.health/report-repair-metrics! {:orphan-count 3})))+ (reset! calls [])+ (memoize/memo-clear! @#'semantic.health/active-index)+ (mt/with-dynamic-fn-redefs [semantic.util/semantic-search-active? (constantly false)]+ (search.index-health/refresh-search-index-metrics!))))+ (is (some (fn [[gauge labels value]]+ (and (= gauge :metabase-search/index-garbage-count)+ (= labels {:index "semantic-search"})+ (double? value) (Double/isNaN ^double value)))+ @calls)+ "the semantic garbage-count series is cleared with NaN"))))diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/migration_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/migration_test.cljindex de40a2b0254..f23671908fd 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/migration_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/migration_test.clj@@ -300,6 +300,21 @@[:= :table_name [:inline table-name]][:= :column_name [:inline column-name]]]}))))+(deftest current-schema-version-adds-health-metric-columns-test+ (mt/with-premium-features #{:semantic-search}+ (semantic.tu/with-test-db-defaults!+ (let [pgvector (semantic.env/get-pgvector-datasource!)+ index-metadata semantic.index-metadata/default-index-metadata]+ (semantic.db.connection/with-migrate-tx [tx]+ (semantic.db.migration/maybe-migrate! tx {:index-metadata index-metadata}))+ (jdbc/execute! pgvector [(str "ALTER TABLE index_metadata "+ "DROP COLUMN repair_orphan_count, "+ "DROP COLUMN repair_snapshot_at")])+ (semantic.db.connection/with-migrate-tx [tx]+ (semantic.db.migration/maybe-migrate! tx {:index-metadata index-metadata}))+ (is (has-column?! pgvector "index_metadata" "repair_orphan_count"))+ (is (has-column?! pgvector "index_metadata" "repair_snapshot_at"))))))+(deftest dynamic-schema-migration-test(mt/with-premium-features #{:semantic-search}(semantic.tu/with-test-db-defaults!diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/repair_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/repair_test.cljindex 72ca9a3f6be..7251f2a8150 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/repair_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/repair_test.clj@@ -5,6 +5,7 @@[honey.sql :as sql][java-time.api :as t][metabase-enterprise.semantic-search.core :as semantic.core]+ [metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource][metabase-enterprise.semantic-search.embedding :as semantic.embedding][metabase-enterprise.semantic-search.env :as semantic.env][metabase-enterprise.semantic-search.index :as semantic.index]@@ -72,11 +73,10 @@(create-test-document "dashboard" 3 "Animal Stats")]](semantic.core/update-index! initial-docs)(semantic.tu/index-all!)- ;; Ensure repair-index! brings index into consistency with new documents et.- ;; Note: card 2 and dashboard 3 are missing - should be deleted- (let [modified-docs [(create-test-document "card" 1 "Dog Training Guide") ; existing - should remain- (create-test-document "card" 4 "Bird Watching Tips") ; new - should be added- (create-test-document "dashboard" 5 "Wildlife Dashboard")]] ; new - should be added+ ;; Retain card 1, add card 4 and dashboard 5, and tombstone the missing card 2 and dashboard 3.+ (let [modified-docs [(create-test-document "card" 1 "Dog Training Guide")+ (create-test-document "card" 4 "Bird Watching Tips")+ (create-test-document "dashboard" 5 "Wildlife Dashboard")]](semantic.core/repair-index! modified-docs)(let [gate-contents (gate-table-contents pgvector gate-table)new-card-entry (gate-entry-by-id gate-contents "card_4")@@ -108,6 +108,75 @@(let [gate-contents (gate-table-contents pgvector (:gate-table-name index-metadata))](is (some? (:document (gate-entry-by-id gate-contents "card_1"))))))))))))+(deftest count-stale-orphans-test+ ;; Hermetic: the fn only takes table names, so ad-hoc tables stand in for the index/gate/repair tables.+ (when semantic.db.datasource/db-url+ (let [pgvector (semantic.db.datasource/ensure-initialized-data-source!)+ suffix (System/nanoTime)+ index-t (str "count_orphans_index_" suffix)+ gate-t (str "count_orphans_gate_" suffix)+ repair-t (str "count_orphans_repair_" suffix)+ dlq-t (str "count_orphans_dlq_" suffix)+ exec! (fn [q] (jdbc/execute! pgvector [q]))+ count! (fn [watermark]+ (semantic.repair/count-stale-orphans pgvector index-t gate-t repair-t dlq-t watermark))+ wm-ts (t/offset-date-time "2026-01-01T12:00:00Z")]+ (try+ (exec! (format "CREATE TABLE \"%s\" (model text, model_id text)" index-t))+ (exec! (format (str "CREATE TABLE \"%s\" (id text, model text, model_id text, "+ "document_hash text, gated_at timestamptz)")+ gate-t))+ (exec! (format "CREATE TABLE \"%s\" (model text, model_id text)" repair-t))+ (exec! (format "CREATE TABLE \"%s\" (gate_id text, error_gated_at timestamptz)" dlq-t))+ ;; candidate set: card 1 only+ (exec! (format "INSERT INTO \"%s\" VALUES ('card','1')" repair-t))+ ;; index rows: 1 = still a candidate, 2 = tombstone behind the watermark (stale), 3 = tombstone ahead+ ;; of it (indexer backlog), 4 = gate row gone (survived tombstone cleanup), 5 = live gate row,+ ;; 6/7 = tombstones AT the watermark timestamp with gate ids on either side of the watermark id+ (exec! (format (str "INSERT INTO \"%s\" VALUES "+ "('card','1'),('card','2'),('card','3'),('card','4'),('card','5'),"+ "('card','6'),('card','7')")+ index-t))+ (exec! (format (str "INSERT INTO \"%s\" VALUES "+ "('card_2', 'card','2', NULL, timestamptz '2026-01-01 11:00:00+00'), "+ "('card_3', 'card','3', NULL, timestamptz '2026-01-01 13:00:00+00'), "+ "('card_5', 'card','5', 'hash', timestamptz '2026-01-01 11:00:00+00'), "+ "('card_6', 'card','6', NULL, timestamptz '2026-01-01 12:00:00+00'), "+ "('card_7', 'card','7', NULL, timestamptz '2026-01-01 12:00:00+00')")+ gate-t))+ ;; card 2 is a current failed delete and remains pending; card 3's obsolete DLQ generation must not+ ;; hide it once the watermark passes its current tombstone.+ (exec! (format (str "INSERT INTO \"%s\" VALUES "+ "('card_2', timestamptz '2026-01-01 11:00:00+00'), "+ "('card_3', timestamptz '2026-01-01 10:00:00+00')")+ dlq-t))+ (testing "counts non-candidates whose tombstone the (gated_at, id) watermark has passed, or whose+ gate row is gone"+ ;; Watermark id between card_6 and card_7: cards 4 (gate row gone) and 6 (same timestamp, earlier+ ;; id) count. Card 2 has a current DLQ retry; cards 3 and 7 remain ahead of the watermark.+ (is (= 2 (count! {:indexer_last_seen wm-ts :indexer_last_seen_id "card_6a"}))))+ (testing "a tombstone ahead of the watermark is in-flight backlog, not garbage"+ (is (= 4 (count! {:indexer_last_seen (t/plus wm-ts (t/hours 2)) :indexer_last_seen_id ""}))+ "once the watermark passes them, the same tombstones count"))+ (testing "nil watermark (indexer never ran) counts no tombstones, only gate-row-gone orphans"+ (is (= 1 (count! nil))))+ (testing "a timestamp-only watermark (no id) treats same-timestamp tombstones as pending"+ (is (= 1 (count! {:indexer_last_seen wm-ts}))+ "cards 6/7 are pending by watermark and card 2 by DLQ; only gate-row-gone card 4 counts"))+ (testing "a query failure returns nil rather than failing the repair run"+ (is (nil? (semantic.repair/count-stale-orphans pgvector "no_such_table" gate-t repair-t dlq-t+ {:indexer_last_seen wm-ts}))))+ (finally+ (exec! (format "DROP TABLE IF EXISTS \"%s\", \"%s\", \"%s\", \"%s\""+ index-t gate-t repair-t dlq-t)))))))++(deftest count-stale-orphans-propagates-interruption-test+ (mt/with-dynamic-fn-redefs+ [jdbc/execute-one! (fn [& _] (throw (InterruptedException.)))]+ (is (thrown? InterruptedException+ (semantic.repair/count-stale-orphans+ ::pgvector "index" "gate" "repair" "dlq" {})))))+(deftest repair-table-cleanup-test(testing "The repair table gets cleaned up properly at the end of a repair-index! job"(mt/with-premium-features #{:semantic-search}diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/task/index_repair_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/task/index_repair_test.cljnew file mode 100644index 00000000000..4873a49ddc6--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/task/index_repair_test.clj@@ -0,0 +1,41 @@+(ns metabase-enterprise.semantic-search.task.index-repair-test+ (:require+ [clojure.test :refer :all]+ [metabase-enterprise.semantic-search.core :as semantic-search.core]+ [metabase-enterprise.semantic-search.health :as semantic.health]+ [metabase-enterprise.semantic-search.repair :as semantic.repair]+ [metabase-enterprise.semantic-search.task.index-repair :as index-repair]+ [metabase.search.ingestion :as search.ingestion]+ [metabase.test :as mt]+ [next.jdbc :as jdbc]))++(deftest successful-repair-reports-current-metric-snapshot-test+ (let [reported (atom [])+ snapshot-at ::snapshot-at]+ (mt/with-dynamic-fn-redefs+ [search.ingestion/searchable-documents (constantly [])+ semantic-search.core/repair-index! (constantly {:index-id 11, :orphans 7, :snapshot-at snapshot-at})+ semantic.health/report-repair-metrics! (fn [& args] (swap! reported conj args))]+ (#'index-repair/repair-index!)+ (is (= [[11 {:orphan-count 7, :snapshot-at snapshot-at}]] @reported)))))++(deftest lost-delete-query-failure-invalidates-metric-snapshot-test+ (let [reported (atom [])]+ (mt/with-dynamic-fn-redefs+ [search.ingestion/searchable-documents (constantly [])+ jdbc/execute! (fn [& _] (throw (ex-info "anti-join failed" {})))+ semantic-search.core/repair-index! (fn [& _]+ (semantic.repair/find-lost-deletes+ ::pgvector "gate" "repair"))+ semantic.health/report-repair-metrics! (fn [& args] (swap! reported conj args))]+ (#'index-repair/repair-index!)+ (is (= [[nil]] @reported)))))++(deftest interrupted-repair-invalidates-count-and-propagates-test+ (let [reported (atom [])]+ (mt/with-dynamic-fn-redefs+ [search.ingestion/searchable-documents (constantly [])+ semantic-search.core/repair-index! (fn [& _] (throw (InterruptedException.)))+ semantic.health/report-repair-metrics! (fn [& args] (swap! reported conj args))]+ (is (thrown? InterruptedException (#'index-repair/repair-index!)))+ (is (= [[nil]] @reported)))))diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/util_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/util_test.cljindex 9cff3b055ee..733d2722240 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/util_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/util_test.clj@@ -16,6 +16,13 @@(is (= "\"semantic_search\"" (semantic.util/quote-ident "semantic_search")))(is (= "\"weird\"\"name\"" (semantic.util/quote-ident "weird\"name")))))+(deftest ^:parallel quote-table-test+ (testing "a schema-qualified name quotes each part separately, not the whole thing as one identifier"+ (is (= "\"semantic_search\".\"index_table_1\""+ (semantic.util/quote-table "semantic_search.index_table_1"))))+ (testing "an unqualified name is a single quoted identifier"+ (is (= "\"index_table_1\"" (semantic.util/quote-table "index_table_1")))))+(deftest ^:parallel column-keyword-test(testing "a dotted-name keyword (renders as separate identifiers), never a namespaced one"(is (= :index_table_1.model (semantic.util/column-keyword "index_table_1" "model")))@@ -26,6 +33,22 @@(is (= :index_table_1.model(semantic.util/conflict-target-column "semantic_search.index_table_1" "model")))))+(deftest index-ready?-catalog-query-test+ (let [queries (atom [])]+ (mt/with-dynamic-fn-redefs+ [jdbc/execute-one! (fn [_ query _]+ (swap! queries conj query)+ {:index_ready true})]+ (is (true? (semantic.util/index-ready? ::pgvector "semantic_search.index_1")))+ (is (true? (semantic.util/index-ready? ::pgvector "index_2"))))+ (testing "both catalog queries require PostgreSQL to mark the index ready and valid"+ (doseq [[statement] @queries]+ (is (re-find #"x\.indisready" statement))+ (is (re-find #"x\.indisvalid" statement))))+ (testing "qualified names retain their schema constraint"+ (is (= ["semantic_search" "index_1"] (rest (first @queries))))+ (is (= ["index_2"] (rest (second @queries)))))))+(deftest catalog-lookups-schema-scoping-test(testing "qualified names scope table/index existence checks to their schema; a same-named object inanother schema (e.g. the app db's public) must not satisfy them"