Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Add pgvector store health metrics (#78668)

Chris Truter4fa4f822e6cf
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/db/datasource.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/db/datasource.clj
index bb543f0927f..31b98931275 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/db/datasource.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/db/datasource.clj
@@ -13,6 +13,8 @@
(com.mchange.v2.c3p0 DataSources PooledDataSource)
(java.net URLDecoder)
(java.nio.charset StandardCharsets)
+ (java.sql Connection SQLException)
+ (java.util.concurrent Executor)
(org.postgresql PGProperty)))
(set! *warn-on-reflection* true)
@@ -191,6 +193,39 @@
(throw e)))
(throw (ex-info "Semantic search connection pool is not initialized. Call init-db! first." {}))))
+(def ^:private probe-timeout-params
+ "pgjdbc timeouts, in seconds, for the readiness probe, so a blackholed host can't block it forever."
+ {"connectTimeout" "5"
+ "socketTimeout" "10"})
+
+(def ^:private probe-query-timeout-seconds
+ "Statement timeout for the readiness probe, for a server that accepts the query and never answers."
+ 10)
+
+(defn- probe-jdbc-url
+ "The dedicated pgvector URL with [[probe-timeout-params]] applied.
+ These replace any timeouts on MB_PGVECTOR_DB_URL rather than deferring to them: an operator raises those
+ for long index builds, and a probe left waiting that long strands a thread and a server-side connection
+ every time it is abandoned -- interrupting it can't end a blocking socket read."
+ []
+ (let [[base pairs] (split-query (:jdbc-url (parsed-db-config)))
+ query (concat (for [[k v] pairs
+ :when (not (probe-timeout-params k))]
+ (str k "=" v))
+ (for [[k v] probe-timeout-params]
+ (str k "=" v)))]
+ (cond-> base
+ (seq query) (str "?" (str/join "&" query)))))
+
+(defn probe-dedicated-connection!
+ "Test the dedicated pgvector database, without initializing or borrowing its connection pool.
+ Always a one-shot connection: a pooled one carries the pool's timeouts, and waiting on a saturated pool
+ would read as an unreachable store."
+ []
+ (jdbc/execute-one! (jdbc/get-datasource {:jdbcUrl (probe-jdbc-url)})
+ ["SELECT 1 AS test"]
+ {:timeout probe-query-timeout-seconds}))
+
(comment
;; docker-compose.yml
(.doReset #'db-url "jdbc:postgres://localhost:55432/mb_semantic_search?user=postgres&password=postgres")
@@ -210,6 +245,12 @@
re-query the catalog on every search and 20s indexer tick. Tests reset it."
(atom nil))
+(def app-db-support-check-errored?
+ "Whether the last app-db pgvector support check failed to answer, rather than answering no.
+ Both outcomes read as unsupported, but only the second is a fact: the readiness probe uses this to report
+ that it couldn't find out instead of publishing a guess it would then sit on for an hour. Tests reset it."
+ (atom false))
+
(def ^:private probe-cooldown-ms
"How long an unsupported or failed app-db pgvector probe is trusted before re-probing.
Long enough that a never-provisioned instance isn't running a rolled-back CREATE probe into its DDL audit
@@ -228,21 +269,103 @@
(let [timer @probe-cooldown-timer]
(or (nil? timer) (>= (u/since-ms timer) probe-cooldown-ms))))
+(def ^:private provisioning-denied-sql-states
+ "The SQLStates that answer the provisioning question with a no: this role may not create the pieces that
+ are missing, or this server cannot host them at all."
+ #{"42501" ; insufficient_privilege -- the role lacks CREATE
+ "0A000" ; feature_not_supported -- e.g. extension "vector" is not available on this server
+ "58P01" ; undefined_file -- the extension's control file is missing from the installation
+ "42704"}) ; undefined_object
+
+(defn- exception-chain
+ "Every exception reachable from `e`, following causes and, for a SQLException, its next exception.
+ JDBC hangs sibling errors off the latter, which the usual cause walk never reaches, so the refusal can sit
+ a link off the path [[u/full-exception-chain]] takes."
+ [e]
+ (loop [pending [e], seen #{}, found []]
+ (if-let [^Throwable x (first pending)]
+ (if (seen x)
+ (recur (rest pending) seen found)
+ (recur (into (vec (rest pending))
+ (remove nil?)
+ [(.getCause x)
+ (when (instance? SQLException x) (.getNextException ^SQLException x))])
+ (conj seen x)
+ (conj found x)))
+ found)))
+
+(defn- provisioning-denied?
+ "Whether `e` is the database refusing to provision, rather than failing to answer.
+ Anything else -- a statement timeout, a dropped connection, exhausted resources, interruption -- leaves
+ the question open, and reads as a denial only if we let it."
+ [e]
+ (boolean (some #(and (instance? SQLException %)
+ (contains? provisioning-denied-sql-states (.getSQLState ^SQLException %)))
+ (exception-chain e))))
+
+(def ^:private probe-network-timeout-ms
+ "Socket-level bound on a probe's reads, for a host that accepts the connection and then stops answering.
+ A statement timeout needs a working connection to carry the cancel, so it does nothing there; the readiness
+ probe cannot be interrupted out of a stuck read either, and one left in that state blocks every later
+ refresh for the life of the process."
+ (int (* 15 1000)))
+
+(def ^:private ^Executor same-thread-executor
+ "For [[java.sql.Connection/setNetworkTimeout]], which requires an executor but only runs bookkeeping on it."
+ (reify Executor
+ (execute [_ r] (.run ^Runnable r))))
+
+(defn- network-timeout
+ "The connection's network timeout, or nil where the driver has no opinion."
+ [^Connection conn]
+ (try
+ (.getNetworkTimeout conn)
+ (catch SQLException _ nil)))
+
+(defn- set-network-timeout!
+ [^Connection conn ^long timeout-ms]
+ (try
+ (.setNetworkTimeout conn same-thread-executor timeout-ms)
+ (catch SQLException _
+ ;; Unsupported, or the connection is already broken. Either way the statement timeouts still apply.
+ nil)))
+
+(defn- with-probe-connection
+ "Call `f` with an app-db connection whose reads are bounded by [[probe-network-timeout-ms]].
+ The bound is lifted again before check-in: this is a pooled connection, and c3p0 hands the same physical one
+ to whoever borrows it next, who is entitled to the app db's own patience -- a migration or a `copy-to-h2`
+ would fail in seconds under the probe's. Both calls are local to the socket, so neither can hang.
+ Check-in is therefore unbounded, which is the lesser problem: a connection whose read just timed out is
+ broken, and c3p0 destroys rather than reuses it."
+ [f]
+ (with-open [conn (jdbc/get-connection (mdb/data-source))]
+ (let [restore-to (network-timeout conn)]
+ (set-network-timeout! conn probe-network-timeout-ms)
+ (try
+ (f conn)
+ (finally
+ (when restore-to
+ (set-network-timeout! conn restore-to)))))))
+
(defn- app-db-can-provision-pgvector?
"Whether the app-db user can create whichever store pieces are still missing, checked without persisting
them: the CREATEs run in a transaction that always rolls back.
Attempts CREATE EXTENSION only when `create-extension?` and CREATE SCHEMA only when `create-schema?`, so
an already-installed extension or existing schema needs no create privilege.
- A privilege error reads as false."
- [app-datasource create-extension? create-schema?]
+ A refusal reads as false; a check that never got that far throws, so the caller can tell the two apart."
+ [conn create-extension? create-schema?]
(try
- (jdbc/with-transaction [tx app-datasource {:rollback-only true}]
+ (jdbc/with-transaction [tx conn {:rollback-only true}]
(when create-extension?
- (jdbc/execute! tx ["CREATE EXTENSION IF NOT EXISTS vector"]))
+ (jdbc/execute! tx ["CREATE EXTENSION IF NOT EXISTS vector"]
+ {:timeout probe-query-timeout-seconds}))
(when create-schema?
- (jdbc/execute! tx [(str "CREATE SCHEMA IF NOT EXISTS " (quoted/postgres app-db-schema))])))
+ (jdbc/execute! tx [(str "CREATE SCHEMA IF NOT EXISTS " (quoted/postgres app-db-schema))]
+ {:timeout probe-query-timeout-seconds})))
true
(catch Exception e
+ (when-not (provisioning-denied? e)
+ (throw e))
(log/debugf "Semantic search: the application database user cannot provision the pgvector store: %s"
(ex-message e))
false)))
@@ -255,27 +378,41 @@
pg_available_extensions: managed Postgres often lists the extension as available while denying the DDL.
The probe persists nothing, so the unlicensed and disabled instances whose availability predicates reach
here never mutate the app db; the persisted CREATE EXTENSION / CREATE SCHEMA run only on the activation
- path ([[metabase-enterprise.semantic-search.pgvector-api/init-semantic-search!]])."
+ path ([[metabase-enterprise.semantic-search.pgvector-api/init-semantic-search!]]).
+ Runs on one [[with-probe-connection]], so a host that stops answering ends the check instead of stranding
+ the readiness probe, which cannot be interrupted out of a stuck read."
+ []
+ (with-probe-connection
+ (fn [conn]
+ ;; The schema_exists SQL alias reads information_schema.schemata (privilege-filtered), not
+ ;; pg_namespace. It answers "a semantic_search schema this role can use exists", not mere catalog
+ ;; presence: a schema the app-db role lacks USAGE on reads as absent, so the store degrades to
+ ;; unavailable rather than passing here and crashing later when init creates tables it can't write.
+ (let [{:keys [installed available schema-exists]}
+ (jdbc/execute-one! conn
+ [(str "SELECT"
+ " EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'vector') AS installed,"
+ " EXISTS (SELECT 1 FROM pg_available_extensions WHERE name = 'vector') AS available,"
+ " EXISTS (SELECT 1 FROM information_schema.schemata WHERE schema_name = ?) AS schema_exists")
+ app-db-schema]
+ {:builder-fn jdbc.rs/as-unqualified-kebab-maps
+ :timeout probe-query-timeout-seconds})]
+ (cond
+ (not (or installed available)) false
+ (and installed schema-exists) true
+ :else (app-db-can-provision-pgvector? conn
+ (not installed)
+ (not schema-exists)))))))
+
+(defn probe-app-db-store!
+ "Whether the app db is a usable pgvector store right now: it answers, and it can still host the store.
+ Uncached, so an extension dropped under a running instance shows up despite [[app-db-pgvector-support]]
+ latching true.
+ Asks [[check-app-db-pgvector-support]], the same question [[pgvector-mode]] selects `:app-db` on, so the
+ two can't disagree. Demanding an installed extension instead would report every instance that has not
+ activated semantic search yet -- where nothing has run `CREATE EXTENSION` -- as permanently unreachable."
[]
- (let [app-datasource (mdb/data-source)
- ;; The schema_exists SQL alias reads information_schema.schemata (privilege-filtered), not
- ;; pg_namespace. It answers "a semantic_search schema this role can use exists", not mere catalog
- ;; presence: a schema the app-db role lacks USAGE on reads as absent, so the store degrades to
- ;; unavailable rather than passing here and crashing later when init creates tables it can't write.
- {:keys [installed available schema-exists]}
- (jdbc/execute-one! app-datasource
- [(str "SELECT"
- " EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'vector') AS installed,"
- " EXISTS (SELECT 1 FROM pg_available_extensions WHERE name = 'vector') AS available,"
- " EXISTS (SELECT 1 FROM information_schema.schemata WHERE schema_name = ?) AS schema_exists")
- app-db-schema]
- {:builder-fn jdbc.rs/as-unqualified-kebab-maps})]
- (cond
- (not (or installed available)) false
- (and installed schema-exists) true
- :else (app-db-can-provision-pgvector? app-datasource
- (not installed)
- (not schema-exists)))))
+ (check-app-db-pgvector-support))
(defn- app-db-pgvector-supported?
"Whether the application database can act as the pgvector store, via a cached probe.
@@ -301,6 +438,9 @@
" (MB_PGVECTOR_DB_URL is not set). Set it if this instance should use a"
" dedicated pgvector database."))
(reset! app-db-pgvector-support true)
+ ;; Outcome before cooldown, always: a reader takes neither under the lock, and one that
+ ;; saw the new cooldown beside the previous outcome would trust a result we just replaced.
+ (reset! app-db-support-check-errored? false)
(reset! probe-cooldown-timer nil)
true)
(do
@@ -309,9 +449,11 @@
" app-db user cannot create the vector extension or the semantic_search"
" schema). Install pgvector and grant CREATE, or set MB_PGVECTOR_DB_URL;"
" it is picked up automatically, no restart needed.")))
+ (reset! app-db-support-check-errored? false)
(reset! probe-cooldown-timer (u/start-timer))
false))
(catch Exception e
+ (reset! app-db-support-check-errored? true)
(reset! probe-cooldown-timer (u/start-timer))
(log/warn (str "Semantic search: pgvector support check on the application database failed;"
" will retry after the cooldown:")
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/db/store_health.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/db/store_health.clj
new file mode 100644
index 00000000000..edfbe2f7fa5
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/db/store_health.clj
@@ -0,0 +1,243 @@
+(ns metabase-enterprise.semantic-search.db.store-health
+ "Readiness of the pgvector store itself -- can this instance reach a vector database, and which one.
+ The store is shared: semantic search and entity retrieval each provision their own tables in it, as more
+ may later, so this reports the store rather than any one feature's index.
+ Index health lives in [[metabase-enterprise.semantic-search.health]]."
+ (:require
+ [metabase-enterprise.semantic-search.db.datasource :as semantic.datasource]
+ [metabase-enterprise.semantic-search.util :as semantic.u]
+ [metabase.analytics-interface.core :as analytics]
+ [metabase.analytics.core :as analytics.core]
+ [metabase.app-db.core :as mdb]
+ [metabase.health-inspector.core :as health-inspector]
+ [metabase.search.index-health :as search.index-health]
+ [metabase.util.log :as log])
+ (:import
+ (java.time Instant)))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private storage-labels
+ "The mutually exclusive pgvector backings, as `:storage` label values.
+ A dedicated URL always wins over the app-db fallback, so at most one can be available at a time."
+ ["dedicated" "appdb"])
+
+(defonce ^:private ^{:doc "The last readiness probe, `{:storage :connected? :at}`, nil before the first.
+ Shared with [[pgvector-store-health-check]] so it doesn't probe again."}
+ last-readiness-probe
+ (atom nil))
+
+(def ^:private readiness-refresh-interval-seconds
+ "How often the readiness probe runs.
+ Hourly: an app db that can't host pgvector yet re-runs the support check, which may attempt rolled-back DDL."
+ (* 60 60))
+
+(def ^:private readiness-scrape-interval-seconds
+ "How often the scrape path reconsiders probing.
+ Far shorter than the probe interval, because the pull collector stamps its throttle before the refresh
+ runs: an hourly slot spent on a probe that couldn't answer -- an app db still migrating at startup -- would
+ leave the gauges wrong for the rest of the hour. The probe's own cadence is enforced by
+ [[readiness-refresh-due?]] instead, off the last probe that did answer."
+ 60)
+
+(def ^:private probe-wait-ms
+ "How long the health check waits on a probe before reporting without it.
+ Above the JDBC timeouts that end one first: the dedicated probe's connect and socket timeouts, the app db's
+ connection checkout plus statement timeouts.
+ Only a probe none of those bound runs past it -- a blackholed host on a URL with no socketTimeout -- and
+ holding the whole health-inspector run behind that would be worse than reporting the previous answer."
+ (* 60 1000))
+
+(defn- store-connected?
+ "Whether `mode`'s store answers. A refused or failed connection is an answer, and reads as disconnected."
+ [mode]
+ (try
+ (case mode
+ :dedicated (do (semantic.datasource/probe-dedicated-connection!) true)
+ :app-db (semantic.datasource/probe-app-db-store!)
+ :unavailable false)
+ (catch Exception e
+ ;; The exception is the only record of why the gauge dropped.
+ (log/warn e "Pgvector connection probe failed" {:mode mode})
+ false)))
+
+(defn- app-db-store-allowed?
+ "Whether this instance may check the app db for pgvector support, the same boot-safe gate the semantic
+ metric collector schedules on.
+ Licensed, so an unlicensed instance never runs that check and its rolled-back DDL."
+ []
+ ;; TODO (Chris 2026-07-29) -- `:semantic-search` only.
+ ;; Entity retrieval is dedicated-only today, so its instances never reach here; once it supports pgvector
+ ;; on the app db, an app-db instance licensed for `:library-retrieval` alone would report no store at all.
+ (semantic.u/semantic-search-configured?))
+
+(def ^:private unresolved-store
+ "What the probe reports when it couldn't find out: no store, and due to be retried on the next scrape
+ rather than holding the hourly slot."
+ {:mode :unavailable, :connected? false, :resolved? false})
+
+(defn- resolve-store
+ "Which store this instance uses, and whether it answers."
+ []
+ (cond
+ ;; A dedicated URL always wins, and reading it asks the app db nothing.
+ (semantic.datasource/dedicated-url-configured?)
+ {:mode :dedicated, :connected? (store-connected? :dedicated), :resolved? true}
+
+ ;; Everything below reads the license or probes the app db, and neither can answer until migrations
+ ;; finish. Say so rather than guess, or the guess is what the gauges show for the next hour.
+ (not (mdb/db-is-set-up?))
+ unresolved-store
+
+ (not (app-db-store-allowed?))
+ {:mode :unavailable, :connected? false, :resolved? true}
+
+ :else
+ (let [mode (semantic.datasource/pgvector-mode)]
+ {:mode mode
+ :connected? (store-connected? mode)
+ ;; A support check that errored also reads as :unavailable, and that is a guess, not an answer.
+ :resolved? (or (not= mode :unavailable)
+ (not @semantic.datasource/app-db-support-check-errored?))})))
+
+(defn- probe-store
+ "[[resolve-store]], reporting no store when it throws.
+ Choosing the mode is itself a database call -- the app-db arm can attempt rolled-back DDL against a host
+ that never answers -- so the whole resolution runs here, not only the connection probe."
+ []
+ (try
+ (resolve-store)
+ ;; Throwable, not Exception: an Error here would otherwise abandon the gauge writes, freezing every
+ ;; series at its last value instead of dropping it.
+ (catch Throwable e
+ ;; The exception is the only record of why the gauges dropped.
+ (log/warn e "Pgvector store probe failed")
+ unresolved-store)))
+
+(defn- clear-stale-last-success!
+ "Drop the last-success timestamp when this instance switches from one backing to another.
+ It is only ever written for the current storage, so the label left behind would otherwise sit at its final
+ value for the life of the process, reading as a store that has not connected since.
+ Losing the store is not a switch: that is precisely when the last known good timestamp is worth keeping."
+ [previous-storage storage]
+ (when (and previous-storage storage (not= previous-storage storage))
+ ;; Clearing takes every label with it, which is what we want here -- only one can ever hold a value.
+ (analytics/clear! :metabase-pgvector/store-last-success-timestamp-seconds)))
+
+(defn- collect-pgvector-readiness-metrics!
+ "Record availability and connection health for the selected pgvector store.
+ Ignores engine activation, so a pgvector rollout can be validated before enabling anything that uses it.
+ Needs no lock of its own: [[request-pgvector-readiness-refresh!]] admits one probe at a time."
+ []
+ (let [previous-storage (:storage @last-readiness-probe)
+ {:keys [mode connected? resolved?]} (probe-store)
+ storage (case mode :dedicated "dedicated" :app-db "appdb" nil)
+ at (.getEpochSecond (Instant/now))]
+ ;; Publish both stable series on every instance; both are zero when no store is usable.
+ (doseq [candidate storage-labels]
+ (analytics/set-gauge! :metabase-pgvector/store-available
+ {:storage candidate}
+ (if (= candidate storage) 1 0))
+ (analytics/set-gauge! :metabase-pgvector/store-connected
+ {:storage candidate}
+ (if (and (= candidate storage) connected?) 1 0)))
+ (clear-stale-last-success! previous-storage storage)
+ (reset! last-readiness-probe {:storage storage
+ :connected? connected?
+ :resolved? resolved?
+ :at at})
+ (when connected?
+ (analytics/set-gauge! :metabase-pgvector/store-last-success-timestamp-seconds
+ {:storage storage}
+ at))))
+
+(defn- readiness-probe-stale?
+ [{:keys [at]}]
+ (or (nil? at)
+ (> (- (.getEpochSecond (Instant/now)) ^long at)
+ (* 2 readiness-refresh-interval-seconds))))
+
+(defn- readiness-refresh-due?
+ "Whether to probe again on this scrape.
+ A probe that couldn't answer is retried on the next one; one that did answer holds for the full interval."
+ [{:keys [at resolved?]}]
+ (or (nil? at)
+ (not resolved?)
+ (>= (- (.getEpochSecond (Instant/now)) ^long at) readiness-refresh-interval-seconds)))
+
+(defn- refresh-pgvector-readiness-metrics!
+ []
+ (try
+ (collect-pgvector-readiness-metrics!)
+ (catch Throwable e
+ ;; The scrape path never derefs this, so a throw here is otherwise silent.
+ (log/error e "Pgvector readiness metric refresh failed"))))
+
+(defonce ^:private readiness-probe-future (atom nil))
+
+(defn- submit-pgvector-readiness-refresh!
+ [f]
+ (future-call f))
+
+(defn- request-pgvector-readiness-refresh!
+ "Start a readiness probe unless one is still running, and return the running probe.
+ A live probe is never replaced. One stuck in a socket read cannot be cancelled, so admitting a replacement
+ would strand another thread on every scrape; leaving the gauges on their last values until it ends is the
+ better of the two, and only a database that answers nothing at all gets there."
+ []
+ (locking readiness-probe-future
+ (let [running @readiness-probe-future]
+ (if (and running (not (realized? running)))
+ running
+ (reset! readiness-probe-future
+ (submit-pgvector-readiness-refresh! refresh-pgvector-readiness-metrics!))))))
+
+(defn- pgvector-store-health-check
+ "Health-inspector row for pgvector store reachability, nil when there is no store to reach.
+ Reports the last probe, waiting up to [[probe-wait-ms]] on a fresh one when that is stale, so an unscraped
+ instance still gets a current row without holding up every other check behind an unresponsive database."
+ []
+ (when (readiness-probe-stale? @last-readiness-probe)
+ (deref (request-pgvector-readiness-refresh!) probe-wait-ms nil))
+ (let [{:keys [storage connected? resolved?] :as probe} @last-readiness-probe]
+ (cond
+ ;; nil is reserved for "this instance has no pgvector store", which is a fact. Not finding out is not,
+ ;; whether the probe answered that it couldn't tell or is still out there when the wait runs out.
+ (or (nil? probe) (not resolved?))
+ (search.index-health/degraded "Could not determine whether a pgvector store is reachable.")
+
+ (not storage) nil
+ connected? (search.index-health/healthy (format "pgvector store (%s) reachable." storage))
+ :else (search.index-health/degraded (format "pgvector store (%s) unreachable." storage)))))
+
+(health-inspector/register-check! :pgvector-store pgvector-store-health-check)
+
+;; Seed both series at startup so every process exposes them before its first probe.
+(defmethod analytics.core/known-labels :metabase-pgvector/store-available [_]
+ (for [storage storage-labels]
+ {:storage storage}))
+
+(defmethod analytics.core/known-labels :metabase-pgvector/store-connected [_]
+ (analytics.core/known-labels :metabase-pgvector/store-available))
+
+;; store-last-success-timestamp-seconds is deliberately absent: a seeded 0 reads as 1970, firing any
+;; staleness alert forever.
+;; It appears once a probe succeeds.
+
+;; No initial-value either, though a configured dedicated URL is available without being probed:
+;; observe-initial-values increments rather than sets, so seeding 1 would leave the gauge at 2 wherever a
+;; scrape had already probed. Both series start at 0 and the first probe corrects them.
+
+(defn- scrape-pgvector-readiness-gauges!
+ "Probe when due, and never wait for the answer -- the scrape carries no database latency."
+ []
+ (when (readiness-refresh-due? @last-readiness-probe)
+ (request-pgvector-readiness-refresh!))
+ nil)
+
+;; Prometheus scrapes every Metabase process, whereas a Quartz job runs on only one member of a cluster.
+;; Start a local, single-flight background probe from the scrape path so every process refreshes its own
+;; gauge series.
+(defmethod analytics.core/pull-collector ::pgvector-readiness-gauges [_]
+ {:min-interval-s readiness-scrape-interval-seconds
+ :f scrape-pgvector-readiness-gauges!})
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/init.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/init.clj
index 61d95f9e33f..412f2788e63 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/init.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/init.clj
@@ -1,5 +1,6 @@
(ns metabase-enterprise.semantic-search.init
(:require
+ [metabase-enterprise.semantic-search.db.store-health]
[metabase-enterprise.semantic-search.embedding-health]
[metabase-enterprise.semantic-search.events]
[metabase-enterprise.semantic-search.health]
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/db/datasource_mode_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/db/datasource_mode_test.clj
index ef0b572bd0c..ee8b464b78a 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/db/datasource_mode_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/db/datasource_mode_test.clj
@@ -10,10 +10,40 @@
[metabase.test :as mt]
[next.jdbc :as jdbc])
(:import
- (clojure.lang ExceptionInfo)))
+ (clojure.lang ExceptionInfo)
+ (java.sql Connection)))
(set! *warn-on-reflection* true)
+(defn- stub-connection
+ "The support check borrows a connection to bound it at the socket level; these tests stub every statement
+ on it, so it only has to close, report its timeout, and record the ones it is handed."
+ (^Connection [] (stub-connection 0 (atom [])))
+ (^Connection [initial-ms timeouts]
+ (reify Connection
+ (getNetworkTimeout [_] initial-ms)
+ (setNetworkTimeout [_ _ ms] (swap! timeouts conj ms))
+ (close [_] nil))))
+
+(deftest probe-connection-restores-its-timeout-test
+ (testing "the probe's socket bound is lifted before check-in, so the next borrower keeps the app db's own"
+ (let [with-probe-connection @#'semantic.db.datasource/with-probe-connection
+ probe-bound @#'semantic.db.datasource/probe-network-timeout-ms]
+ (doseq [initial-ms [0 300000]]
+ (testing (format "from an existing timeout of %dms" initial-ms)
+ (let [timeouts (atom [])]
+ (with-redefs [mdb/data-source (constantly ::app-pool)
+ jdbc/get-connection (fn [_] (stub-connection initial-ms timeouts))]
+ (is (= ::probed (with-probe-connection (constantly ::probed))))
+ (is (= [probe-bound initial-ms] @timeouts))))))
+ (testing "and when the probe throws, which is how a stuck read ends"
+ (let [timeouts (atom [])]
+ (with-redefs [mdb/data-source (constantly ::app-pool)
+ jdbc/get-connection (fn [_] (stub-connection 300000 timeouts))]
+ (is (thrown? ExceptionInfo
+ (with-probe-connection (fn [_] (throw (ex-info "stuck" {}))))))
+ (is (= [probe-bound 300000] @timeouts))))))))
+
(defmacro ^:private with-support-cache
"Run body with all three pieces of app-db probe state rebound to fresh atoms: the support cache (holding
`init`), the cooldown timer, and the log-once hint latch."
@@ -57,7 +87,8 @@
(testing "supported only when the vector extension and the semantic_search schema exist or can be created"
(letfn [(check [] (semantic.db.datasource/check-app-db-pgvector-support))
(catalog [m] (fn [& _] m))]
- (with-redefs [mdb/data-source (constantly ::app-pool)]
+ (with-redefs [mdb/data-source (constantly ::app-pool)
+ jdbc/get-connection (fn [_] (stub-connection))]
(testing "extension neither installed nor available → unsupported, no provisioning probe"
(with-redefs [jdbc/execute-one! (catalog {:installed false :available false :schema-exists false})
semantic.db.datasource/app-db-can-provision-pgvector?
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/db/store_health_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/db/store_health_test.clj
new file mode 100644
index 00000000000..02d9fac3206
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/db/store_health_test.clj
@@ -0,0 +1,284 @@
+(ns metabase-enterprise.semantic-search.db.store-health-test
+ (:require
+ [clojure.string :as str]
+ [clojure.test :refer [deftest is testing]]
+ [iapetos.export :as export]
+ [java-time.api :as t]
+ [metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource]
+ [metabase-enterprise.semantic-search.db.store-health :as semantic.store-health]
+ [metabase-enterprise.semantic-search.util :as semantic.u]
+ [metabase.analytics-interface.core :as analytics]
+ [metabase.analytics.core :as analytics.core]
+ [metabase.app-db.core :as mdb]
+ [metabase.test :as mt]))
+
+(defn- readiness-gauges
+ "Every readiness gauge for both storage labels, as `{storage {:available _ :connected _ :last-success _}}`."
+ [system]
+ (into {}
+ (for [storage ["dedicated" "appdb"]]
+ [storage {:available (mt/metric-value system :metabase-pgvector/store-available
+ {:storage storage})
+ :connected (mt/metric-value system :metabase-pgvector/store-connected
+ {:storage storage})
+ :last-success (mt/metric-value
+ system :metabase-pgvector/store-last-success-timestamp-seconds
+ {:storage storage})}])))
+
+(deftest pgvector-readiness-metrics-test
+ (mt/with-prometheus-system! [_ system]
+ (testing "an instance without any pgvector backend is explicitly not ready"
+ (mt/with-dynamic-fn-redefs
+ [semantic.db.datasource/dedicated-url-configured? (constantly false)
+ semantic.db.datasource/pgvector-mode (constantly :unavailable)
+ semantic.db.datasource/probe-dedicated-connection! #(throw (ex-info "should not connect" {}))]
+ (@#'semantic.store-health/collect-pgvector-readiness-metrics!)
+ (is (=? {"dedicated" {:available zero?, :connected zero?}
+ "appdb" {:available zero?, :connected zero?}}
+ (readiness-gauges system)))))
+ (testing "a connected dedicated store is available and healthy"
+ (mt/with-dynamic-fn-redefs
+ [semantic.db.datasource/dedicated-url-configured? (constantly true)
+ semantic.db.datasource/probe-dedicated-connection! (constantly {:one 1})]
+ (@#'semantic.store-health/collect-pgvector-readiness-metrics!))
+ (is (=? {"dedicated" {:available #(== 1 %), :connected #(== 1 %), :last-success pos?}
+ "appdb" {:available zero?, :connected zero?}}
+ (readiness-gauges system)))
+ (let [last-success (get-in (readiness-gauges system) ["dedicated" :last-success])]
+ (testing "a later failure clears connected but preserves the last-success timestamp"
+ (mt/with-dynamic-fn-redefs
+ [semantic.db.datasource/dedicated-url-configured? (constantly true)
+ semantic.db.datasource/probe-dedicated-connection! #(throw (ex-info "probe failed" {}))]
+ (@#'semantic.store-health/collect-pgvector-readiness-metrics!))
+ (is (=? {"dedicated" {:available #(== 1 %), :connected zero?, :last-success #(== last-success %)}}
+ (readiness-gauges system))))))
+ (testing "a connected app-db pgvector store is available and healthy"
+ (mt/with-dynamic-fn-redefs
+ [mdb/db-is-set-up? (constantly true)
+ semantic.db.datasource/dedicated-url-configured? (constantly false)
+ semantic.u/semantic-search-configured? (constantly true)
+ semantic.db.datasource/pgvector-mode (constantly :app-db)
+ semantic.db.datasource/probe-app-db-store! (constantly true)]
+ (@#'semantic.store-health/collect-pgvector-readiness-metrics!))
+ (is (=? {"dedicated" {:available zero?, :connected zero?}
+ "appdb" {:available #(== 1 %), :connected #(== 1 %), :last-success pos?}}
+ (readiness-gauges system))))
+ (testing "an app-db store whose vector extension went away reads available but disconnected"
+ (mt/with-dynamic-fn-redefs
+ [mdb/db-is-set-up? (constantly true)
+ semantic.db.datasource/dedicated-url-configured? (constantly false)
+ semantic.u/semantic-search-configured? (constantly true)
+ semantic.db.datasource/pgvector-mode (constantly :app-db)
+ semantic.db.datasource/probe-app-db-store! (constantly false)]
+ (@#'semantic.store-health/collect-pgvector-readiness-metrics!))
+ (is (=? {"appdb" {:available #(== 1 %), :connected zero?}}
+ (readiness-gauges system))))))
+
+(def ^:private last-success-sample-re
+ #"^metabase_pgvector_store_last_success_timestamp_seconds\{storage=\"([^\"]+)\",?\} (\S+)")
+
+(defn- exposed-last-success
+ "The last-success samples a scrape would actually see, as `{storage value}`.
+ Read from the exposition rather than [[mt/metric-value]], which creates the child it looks up and so
+ reports an absent series as a present zero."
+ [system]
+ (into {}
+ (keep (fn [line]
+ (when-let [[_ storage value] (re-find last-success-sample-re line)]
+ [storage (parse-double value)])))
+ (str/split-lines (export/text-format (:registry system)))))
+
+(deftest last-success-does-not-outlive-its-storage-test
+ (mt/with-prometheus-system! [_ system]
+ ;; Scraping runs the pull collectors, this namespace's included. Stub the submit for the whole test so a
+ ;; read of the exposition can't spawn a real probe that races it by writing the series being read.
+ (mt/with-dynamic-fn-redefs
+ [semantic.store-health/submit-pgvector-readiness-refresh! (constantly nil)]
+ (mt/with-dynamic-fn-redefs
+ [semantic.db.datasource/dedicated-url-configured? (constantly true)
+ semantic.db.datasource/probe-dedicated-connection! (constantly {:one 1})]
+ (@#'semantic.store-health/collect-pgvector-readiness-metrics!))
+ (is (=? {"dedicated" pos?} (exposed-last-success system)))
+ (testing "switching storage drops the old timestamp, which would otherwise never advance again"
+ (mt/with-dynamic-fn-redefs
+ [mdb/db-is-set-up? (constantly true)
+ semantic.db.datasource/dedicated-url-configured? (constantly false)
+ semantic.u/semantic-search-configured? (constantly true)
+ semantic.db.datasource/pgvector-mode (constantly :app-db)
+ semantic.db.datasource/probe-app-db-store! (constantly true)]
+ (@#'semantic.store-health/collect-pgvector-readiness-metrics!))
+ (is (=? {"appdb" pos?} (exposed-last-success system)))
+ (is (= #{"appdb"} (set (keys (exposed-last-success system))))
+ "the abandoned label is gone rather than zeroed -- a 0 would read as 1970"))
+ (testing "losing the store keeps the timestamp: that is when it is worth knowing"
+ (mt/with-dynamic-fn-redefs
+ [mdb/db-is-set-up? (constantly true)
+ semantic.db.datasource/dedicated-url-configured? (constantly false)
+ semantic.u/semantic-search-configured? (constantly false)]
+ (@#'semantic.store-health/collect-pgvector-readiness-metrics!))
+ (is (=? {"appdb" pos?} (exposed-last-success system)))))))
+
+(deftest probe-store-test
+ (let [probe @#'semantic.store-health/probe-store]
+ (mt/with-dynamic-fn-redefs [semantic.db.datasource/dedicated-url-configured? (constantly true)]
+ (testing "an ordinary failure reads as disconnected, but the store is still known"
+ (mt/with-dynamic-fn-redefs
+ [semantic.db.datasource/probe-dedicated-connection! #(throw (ex-info "nope" {}))]
+ (is (=? {:mode :dedicated, :connected? false, :resolved? true} (probe)))))
+ (testing "an Error does not escape and abandon the gauge writes"
+ (mt/with-dynamic-fn-redefs
+ [semantic.db.datasource/probe-dedicated-connection! #(throw (AssertionError. "boom"))]
+ (is (=? {:mode :unavailable, :connected? false, :resolved? false} (probe)))))
+ (testing "a store that answers reads as connected"
+ (mt/with-dynamic-fn-redefs
+ [semantic.db.datasource/probe-dedicated-connection! (constantly {:one 1})]
+ (is (=? {:mode :dedicated, :connected? true, :resolved? true} (probe))))))
+ (testing "a support check that errored is unresolved: unsupported and unknown both read as :unavailable"
+ (reset! semantic.db.datasource/app-db-support-check-errored? true)
+ (try
+ (mt/with-dynamic-fn-redefs
+ [mdb/db-is-set-up? (constantly true)
+ semantic.db.datasource/dedicated-url-configured? (constantly false)
+ semantic.u/semantic-search-configured? (constantly true)
+ semantic.db.datasource/pgvector-mode (constantly :unavailable)]
+ (is (=? {:mode :unavailable, :connected? false, :resolved? false} (probe)))
+ (testing "one that answered no is an answer, and holds for the interval"
+ (reset! semantic.db.datasource/app-db-support-check-errored? false)
+ (is (=? {:mode :unavailable, :connected? false, :resolved? true} (probe)))))
+ (finally
+ (reset! semantic.db.datasource/app-db-support-check-errored? false))))
+ (testing "an app db that is not migrated yet is unresolved, not an answer to cache for an hour"
+ (mt/with-dynamic-fn-redefs
+ [mdb/db-is-set-up? (constantly false)
+ semantic.db.datasource/dedicated-url-configured? (constantly false)
+ semantic.db.datasource/pgvector-mode #(throw (ex-info "must not probe" {}))]
+ (is (=? {:mode :unavailable, :connected? false, :resolved? false} (probe)))))))
+
+(deftest ^:sequential live-probe-is-never-replaced-test
+ (testing "a probe stuck on an unresponsive database cannot be cancelled, so no replacement is admitted"
+ (let [probe-future @#'semantic.store-health/readiness-probe-future
+ request @#'semantic.store-health/request-pgvector-readiness-refresh!
+ started (promise)
+ release (promise)
+ starts (atom 0)]
+ (try
+ (reset! probe-future nil)
+ (mt/with-dynamic-fn-redefs
+ [semantic.store-health/refresh-pgvector-readiness-metrics!
+ ;; Counted before it is announced, so awaiting `started` proves the count is in.
+ (fn [] (swap! starts inc) (deliver started true) (deref release 10000 ::hung))]
+ (let [first-probe (request)]
+ (is (identical? first-probe (request)) "the running probe is returned rather than replaced")
+ ;; The request only submits, so wait for the body rather than racing the scheduler.
+ (is (true? (deref started 10000 ::never-ran)))
+ (is (= 1 @starts) "and no second one is started")
+ (deliver release true)
+ (is (true? (deref first-probe 10000 ::hung)))
+ (testing "once it ends, the next request starts a fresh one"
+ (let [next-probe (request)]
+ (is (not (identical? first-probe next-probe)))
+ (is (true? (deref next-probe 10000 ::hung)))
+ (is (= 2 @starts))))))
+ (finally
+ (deliver release true)
+ (reset! probe-future nil))))))
+
+(deftest readiness-refresh-due-test
+ (let [due? @#'semantic.store-health/readiness-refresh-due?
+ now (quot (t/to-millis-from-epoch (t/instant)) 1000)]
+ (is (true? (due? nil)) "nothing probed yet")
+ (is (false? (due? {:at now, :resolved? true})) "a fresh answer holds the interval")
+ (is (true? (due? {:at now, :resolved? false}))
+ "a probe that could not answer is retried rather than holding the hourly slot")
+ (is (true? (due? {:at (- now 3601), :resolved? true})) "the interval has passed")))
+
+(deftest pgvector-readiness-skips-unlicensed-app-db-probe-test
+ (testing "an unlicensed instance never resolves pgvector-mode, whose app-db arm probes the app db"
+ (mt/with-premium-features #{}
+ (mt/with-prometheus-system! [_ system]
+ (mt/with-dynamic-fn-redefs
+ [mdb/db-is-set-up? (constantly true)
+ semantic.db.datasource/dedicated-url-configured? (constantly false)
+ semantic.db.datasource/pgvector-mode #(throw (ex-info "must not probe" {}))]
+ (@#'semantic.store-health/collect-pgvector-readiness-metrics!)
+ (is (=? {"dedicated" {:available zero?, :connected zero?}
+ "appdb" {:available zero?, :connected zero?}}
+ (readiness-gauges system))))))))
+
+(deftest initial-values-test
+ (testing "both storage series are seeded at startup, and the timestamp deliberately is not"
+ (is (= [{:storage "dedicated"} {:storage "appdb"}]
+ (analytics.core/known-labels :metabase-pgvector/store-available)
+ (analytics.core/known-labels :metabase-pgvector/store-connected)))
+ (is (not (contains? (methods analytics.core/known-labels)
+ :metabase-pgvector/store-last-success-timestamp-seconds))
+ "a seeded 0 would read as 1970 and fire any staleness alert forever"))
+ (testing "every series is seeded at 0, even one a configured dedicated URL already makes available"
+ (mt/with-dynamic-fn-redefs [semantic.db.datasource/dedicated-url-configured? (constantly true)]
+ (doseq [storage ["dedicated" "appdb"]]
+ (is (zero? (analytics.core/initial-value :metabase-pgvector/store-available {:storage storage}))
+ "observe-initial-values increments, so a non-zero seed doubles a gauge a scrape already set")))))
+
+(defn- fresh-probe
+ [m]
+ (assoc m :at (quot (t/to-millis-from-epoch (t/instant)) 1000)))
+
+(deftest pgvector-store-health-check-test
+ (let [check @#'semantic.store-health/pgvector-store-health-check
+ probe @#'semantic.store-health/last-readiness-probe
+ collects (atom 0)]
+ (mt/with-dynamic-fn-redefs
+ [semantic.store-health/collect-pgvector-readiness-metrics! #(swap! collects inc)]
+ (testing "a fresh probe is reported as-is, without running another"
+ (reset! probe (fresh-probe {:storage nil, :connected? false, :resolved? true}))
+ (is (nil? (check)) "omitted when there is no store to reach")
+ (reset! probe (fresh-probe {:storage "dedicated", :connected? true, :resolved? true}))
+ (is (=? {:health 100, :message #"pgvector store \(dedicated\) reachable\."} (check)))
+ (reset! probe (fresh-probe {:storage "appdb", :connected? false, :resolved? true}))
+ (is (=? {:health 0, :message #"pgvector store \(appdb\) unreachable\."} (check)))
+ (is (zero? @collects)))
+ (testing "a probe that could not answer is degraded, not omitted as inapplicable"
+ (reset! probe (fresh-probe {:storage nil, :connected? false, :resolved? false}))
+ (is (=? {:health 0, :message #"Could not determine whether a pgvector store is reachable\."} (check))))
+ (testing "a missing or stale probe is refreshed, so an unscraped instance still reports"
+ (reset! probe nil)
+ ;; The refresh is stubbed, so the atom is still empty when the wait ends -- the case where a probe
+ ;; is still out there, which must not read as an absent store.
+ (is (=? {:health 0, :message #"Could not determine whether a pgvector store is reachable\."} (check)))
+ (reset! probe {:storage "dedicated", :connected? true, :resolved? true, :at 0})
+ (check)
+ (is (= 2 @collects))))))
+
+(deftest ^:sequential pull-collector-refreshes-pgvector-metrics-on-each-instance-test
+ (let [probe-future @#'semantic.store-health/readiness-probe-future
+ probe @#'semantic.store-health/last-readiness-probe
+ prior-probe @probe
+ collector (analytics.core/pull-collector ::semantic.store-health/pgvector-readiness-gauges)
+ gauge-calls (atom [])
+ refreshes (atom 0)
+ submitted (atom [])
+ ;; The submitted probe has to stay unrealized, or the second scrape sees a finished one and starts
+ ;; its own -- which is the single-flight behaviour under test.
+ hold (promise)]
+ (try
+ (reset! probe-future nil)
+ ;; Nothing probed yet, so the refresh is due.
+ (reset! probe nil)
+ (mt/with-dynamic-fn-redefs
+ [analytics/set-gauge! #(swap! gauge-calls conj (vec %&))
+ semantic.store-health/collect-pgvector-readiness-metrics! #(swap! refreshes inc)
+ semantic.store-health/submit-pgvector-readiness-refresh! #(do (swap! submitted conj %)
+ (future (deref hold 10000 ::hung)))]
+ ((:f collector))
+ ((:f collector))
+ (is (= 1 (count @submitted)) "overlapping scrapes share one local refresh")
+ (is (empty? @gauge-calls) "the scrape path writes no gauges of its own")
+ (is (zero? @refreshes) "the database probe does not run on the synchronous scrape path")
+ ((first @submitted)))
+ (is (= 60 (:min-interval-s collector))
+ "the scrape reconsiders often; the hourly cadence is readiness-refresh-due?'s job")
+ (is (= 1 @refreshes))
+ (finally
+ (deliver hold true)
+ (reset! probe-future nil)
+ (reset! probe prior-probe)))))
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/db_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/db_test.clj
index c9ecaa76be6..fe771274cdd 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/db_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/db_test.clj
@@ -3,7 +3,9 @@
[clojure.test :refer :all]
[metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource])
(:import
- (com.mchange.v2.c3p0 PoolBackedDataSource)))
+ (com.mchange.v2.c3p0 PoolBackedDataSource)
+ (java.sql SQLException SQLTimeoutException)
+ (javax.sql DataSource)))
(set! *warn-on-reflection* true)
@@ -49,10 +51,35 @@
(finally
(reset! semantic.db.datasource/data-source orig-data-source))))))
+(deftest probe-dedicated-connection-test
+ (testing "the readiness probe connects without initializing the dedicated pool"
+ (when semantic.db.datasource/db-url
+ (with-redefs [semantic.db.datasource/data-source (atom nil)]
+ (is (= {:test 1} (semantic.db.datasource/probe-dedicated-connection!)))
+ (is (nil? @semantic.db.datasource/data-source))))))
+
(def ^:private parse-db-url #'semantic.db.datasource/parse-db-url)
+(def ^:private probe-jdbc-url #'semantic.db.datasource/probe-jdbc-url)
+
(def ^:private base-url "jdbc:postgresql://localhost:5432/mb_semantic_search")
+(deftest probe-jdbc-url-test
+ (testing "the probe URL fills in the fail-fast timeouts the one-shot datasource would otherwise lack"
+ ;; db-url is a value, not a fn, so with-dynamic-fn-redefs can't bind it
+ (with-redefs [semantic.db.datasource/db-url base-url]
+ (is (= (str base-url "?connectTimeout=5&socketTimeout=10")
+ (probe-jdbc-url)))))
+ (testing "a timeout the operator set on MB_PGVECTOR_DB_URL is replaced, not deferred to"
+ ;; An abandoned probe can't be interrupted out of a socket read, so it must not inherit a long wait.
+ (with-redefs [semantic.db.datasource/db-url (str base-url "?socketTimeout=60")]
+ (is (= (str base-url "?connectTimeout=5&socketTimeout=10")
+ (probe-jdbc-url)))))
+ (testing "unrelated connection params pass through, in the order they were written"
+ (with-redefs [semantic.db.datasource/db-url (str base-url "?user=mb&sslmode=require")]
+ (is (= (str base-url "?user=mb&sslmode=require&connectTimeout=5&socketTimeout=10")
+ (probe-jdbc-url))))))
+
(deftest parse-db-url-defaults-test
(testing "a URL with no params leaves the URL untouched and uses the default pool props"
;; strict = (not =?) so an unexpected extra/missing prop also fails, not just a wrong value
@@ -142,3 +169,38 @@
(is (= "metabase-semantic-search-db" (.getDataSourceName pool-ds))))
(finally
(semantic.db.datasource/shutdown-db!))))))))
+
+(def ^:private can-provision? #'semantic.db.datasource/app-db-can-provision-pgvector?)
+
+(defn- failing-datasource
+ "A datasource whose every connection attempt throws `e`, so the provisioning check fails before it can
+ ask the question."
+ ^DataSource [^Exception e]
+ (reify DataSource
+ (getConnection [_] (throw e))
+ (getConnection [_ _ _] (throw e))))
+
+(deftest app-db-can-provision-pgvector-distinguishes-refusal-from-failure-test
+ (testing "the database refusing to provision is an answer, and reads as no"
+ (doseq [[state what] {"42501" "the role lacks CREATE"
+ "0A000" "the extension is not available on this server"
+ "58P01" "the extension's control file is missing"}]
+ (testing what
+ ;; Hinted: SQLException also has a (String, Throwable) ctor, so untyped args reflect.
+ (is (false? (can-provision? (failing-datasource (SQLException. ^String what ^String state))
+ true true))))))
+ (testing "a check that never got an answer throws, rather than reading as a settled no"
+ (doseq [[e what] {(SQLTimeoutException. "statement timeout" "57014") "a timed-out DDL probe"
+ (SQLException. "connection refused" "08006") "a dropped connection"
+ (SQLException. "too many connections" "53300") "an exhausted server"
+ (InterruptedException. "abandoned") "an interrupted probe"}]
+ (testing what
+ (is (thrown? Exception (can-provision? (failing-datasource e) true true))))))
+ (testing "a refusal wrapped in another exception is still recognised"
+ (let [wrapped (SQLException. "rollback failed" "25P02" (SQLException. "denied" "42501"))]
+ (is (false? (can-provision? (failing-datasource wrapped) true true)))))
+ (testing "a refusal hung off getNextException is still recognised"
+ ;; JDBC chains sibling errors there rather than through getCause, so a cause-only walk misses it.
+ (let [chained (doto (SQLException. "batch failed" "25P02")
+ (.setNextException (SQLException. "denied" "42501")))]
+ (is (false? (can-provision? (failing-datasource chained) true true))))))
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/task/metric_collector_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/task/metric_collector_test.clj
index a5cf3401da5..21542f73510 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/task/metric_collector_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/task/metric_collector_test.clj
@@ -70,8 +70,8 @@
(deftest shared-index-metrics-survive-semantic-collector-failure-test
(let [refreshes (atom 0)]
(mt/with-dynamic-fn-redefs
- [semantic.u/semantic-search-active? (constantly true)
- semantic.env/get-pgvector-datasource! #(throw (ex-info "pgvector unavailable" {}))
+ [semantic.u/semantic-search-active? (constantly true)
+ semantic.env/get-pgvector-datasource! #(throw (ex-info "pgvector unavailable" {}))
search.index-health/refresh-search-index-metrics! #(swap! refreshes inc)]
(@#'semantic.task.collector/collect-metrics!)
(is (= 1 @refreshes)))))
@@ -79,8 +79,8 @@
(deftest interrupted-semantic-collector-skips-shared-refresh-test
(let [refreshes (atom 0)]
(mt/with-dynamic-fn-redefs
- [semantic.u/semantic-search-active? (constantly true)
- semantic.env/get-pgvector-datasource! #(throw (InterruptedException.))
+ [semantic.u/semantic-search-active? (constantly true)
+ semantic.env/get-pgvector-datasource! #(throw (InterruptedException.))
search.index-health/refresh-search-index-metrics! #(swap! refreshes inc)]
(is (thrown? InterruptedException (@#'semantic.task.collector/collect-metrics!)))
(is (zero? @refreshes)))))
@@ -91,13 +91,14 @@
(let [pgvector (semantic.env/get-pgvector-datasource!)
index-metadata (semantic.tu/unique-index-metadata)
model semantic.tu/mock-embedding-model]
- (mt/with-dynamic-fn-redefs [semantic.env/get-index-metadata (fn [] index-metadata)
- semantic.env/get-configured-embedding-model (fn [] model)
- ;; supported? requires a configured embedder for engine selection
- semantic.embedding/get-configured-model (fn [] model)
- ;; collect-metrics! only runs when semantic is the active engine; pin it so a
- ;; sibling test leaking the search-engine setting can't make this read 0
- semantic.u/semantic-search-active? (fn [] true)]
+ (mt/with-dynamic-fn-redefs
+ [semantic.env/get-index-metadata (constantly index-metadata)
+ semantic.env/get-configured-embedding-model (constantly model)
+ ;; supported? requires a configured embedder for engine selection
+ semantic.embedding/get-configured-model (constantly model)
+ ;; collect-metrics! only runs when semantic is the active engine; pin it so a sibling test
+ ;; leaking the search-engine setting can't make this read 0
+ semantic.u/semantic-search-active? (constantly true)]
(testing "Missing tables are handled gracefully"
(let [result (try
(@#'semantic.task.collector/collect-metrics!)
diff --git a/src/metabase/analytics/prometheus.clj b/src/metabase/analytics/prometheus.clj
index b6a3390ea6e..6b438c72706 100644
--- a/src/metabase/analytics/prometheus.clj
+++ b/src/metabase/analytics/prometheus.clj
@@ -331,6 +331,21 @@
(prometheus/counter :metabase-remote-sync/git-operations-failed
{:description "Number of failed git operations"
:labels [:operation :remote]})
+ ;; Shared: semantic search and entity retrieval each provision their own tables in it, as more may later.
+ ;; At most one is available at a time: a dedicated MB_PGVECTOR_DB_URL always wins over the app db.
+ (prometheus/gauge :metabase-pgvector/store-available
+ {:description "Whether the given pgvector storage is available to this instance."
+ :labels [:storage]})
+ ;; Probed hourly, so an outage takes up to that long to show up. The dedicated probe opens its own
+ ;; connection, so a store that is up but whose pool is saturated still reads as connected --
+ ;; metabase_database_c3p0_* covers that. The app-db probe shares the application pool, and does not.
+ (prometheus/gauge :metabase-pgvector/store-connected
+ {:description "Whether the last connection probe to the given pgvector storage succeeded."
+ :labels [:storage]})
+ ;; Absent until a probe succeeds, and reset when the instance changes which backing it uses.
+ (prometheus/gauge :metabase-pgvector/store-last-success-timestamp-seconds
+ {:description "Unix timestamp of the last successful probe to the given pgvector storage."
+ :labels [:storage]})
(prometheus/counter :metabase-search/index-reindexes
{:description "Number of reindexed search entries"
:labels [:model]})