The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Add pgvector store health metrics (#78668)
Chris Truter4fa4f822e6cf
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/db/datasource.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/db/datasource.cljindex bb543f0927f..31b98931275 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/db/datasource.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/db/datasource.clj@@ -13,6 +13,8 @@(com.mchange.v2.c3p0 DataSources PooledDataSource)(java.net URLDecoder)(java.nio.charset StandardCharsets)+ (java.sql Connection SQLException)+ (java.util.concurrent Executor)(org.postgresql PGProperty)))(set! *warn-on-reflection* true)@@ -191,6 +193,39 @@(throw e)))(throw (ex-info "Semantic search connection pool is not initialized. Call init-db! first." {}))))+(def ^:private probe-timeout-params+ "pgjdbc timeouts, in seconds, for the readiness probe, so a blackholed host can't block it forever."+ {"connectTimeout" "5"+ "socketTimeout" "10"})++(def ^:private probe-query-timeout-seconds+ "Statement timeout for the readiness probe, for a server that accepts the query and never answers."+ 10)++(defn- probe-jdbc-url+ "The dedicated pgvector URL with [[probe-timeout-params]] applied.+ These replace any timeouts on MB_PGVECTOR_DB_URL rather than deferring to them: an operator raises those+ for long index builds, and a probe left waiting that long strands a thread and a server-side connection+ every time it is abandoned -- interrupting it can't end a blocking socket read."+ []+ (let [[base pairs] (split-query (:jdbc-url (parsed-db-config)))+ query (concat (for [[k v] pairs+ :when (not (probe-timeout-params k))]+ (str k "=" v))+ (for [[k v] probe-timeout-params]+ (str k "=" v)))]+ (cond-> base+ (seq query) (str "?" (str/join "&" query)))))++(defn probe-dedicated-connection!+ "Test the dedicated pgvector database, without initializing or borrowing its connection pool.+ Always a one-shot connection: a pooled one carries the pool's timeouts, and waiting on a saturated pool+ would read as an unreachable store."+ []+ (jdbc/execute-one! (jdbc/get-datasource {:jdbcUrl (probe-jdbc-url)})+ ["SELECT 1 AS test"]+ {:timeout probe-query-timeout-seconds}))+(comment;; docker-compose.yml(.doReset #'db-url "jdbc:postgres://localhost:55432/mb_semantic_search?user=postgres&password=postgres")@@ -210,6 +245,12 @@re-query the catalog on every search and 20s indexer tick. Tests reset it."(atom nil))+(def app-db-support-check-errored?+ "Whether the last app-db pgvector support check failed to answer, rather than answering no.+ Both outcomes read as unsupported, but only the second is a fact: the readiness probe uses this to report+ that it couldn't find out instead of publishing a guess it would then sit on for an hour. Tests reset it."+ (atom false))+(def ^:private probe-cooldown-ms"How long an unsupported or failed app-db pgvector probe is trusted before re-probing.Long enough that a never-provisioned instance isn't running a rolled-back CREATE probe into its DDL audit@@ -228,21 +269,103 @@(let [timer @probe-cooldown-timer](or (nil? timer) (>= (u/since-ms timer) probe-cooldown-ms))))+(def ^:private provisioning-denied-sql-states+ "The SQLStates that answer the provisioning question with a no: this role may not create the pieces that+ are missing, or this server cannot host them at all."+ #{"42501" ; insufficient_privilege -- the role lacks CREATE+ "0A000" ; feature_not_supported -- e.g. extension "vector" is not available on this server+ "58P01" ; undefined_file -- the extension's control file is missing from the installation+ "42704"}) ; undefined_object++(defn- exception-chain+ "Every exception reachable from `e`, following causes and, for a SQLException, its next exception.+ JDBC hangs sibling errors off the latter, which the usual cause walk never reaches, so the refusal can sit+ a link off the path [[u/full-exception-chain]] takes."+ [e]+ (loop [pending [e], seen #{}, found []]+ (if-let [^Throwable x (first pending)]+ (if (seen x)+ (recur (rest pending) seen found)+ (recur (into (vec (rest pending))+ (remove nil?)+ [(.getCause x)+ (when (instance? SQLException x) (.getNextException ^SQLException x))])+ (conj seen x)+ (conj found x)))+ found)))++(defn- provisioning-denied?+ "Whether `e` is the database refusing to provision, rather than failing to answer.+ Anything else -- a statement timeout, a dropped connection, exhausted resources, interruption -- leaves+ the question open, and reads as a denial only if we let it."+ [e]+ (boolean (some #(and (instance? SQLException %)+ (contains? provisioning-denied-sql-states (.getSQLState ^SQLException %)))+ (exception-chain e))))++(def ^:private probe-network-timeout-ms+ "Socket-level bound on a probe's reads, for a host that accepts the connection and then stops answering.+ A statement timeout needs a working connection to carry the cancel, so it does nothing there; the readiness+ probe cannot be interrupted out of a stuck read either, and one left in that state blocks every later+ refresh for the life of the process."+ (int (* 15 1000)))++(def ^:private ^Executor same-thread-executor+ "For [[java.sql.Connection/setNetworkTimeout]], which requires an executor but only runs bookkeeping on it."+ (reify Executor+ (execute [_ r] (.run ^Runnable r))))++(defn- network-timeout+ "The connection's network timeout, or nil where the driver has no opinion."+ [^Connection conn]+ (try+ (.getNetworkTimeout conn)+ (catch SQLException _ nil)))++(defn- set-network-timeout!+ [^Connection conn ^long timeout-ms]+ (try+ (.setNetworkTimeout conn same-thread-executor timeout-ms)+ (catch SQLException _+ ;; Unsupported, or the connection is already broken. Either way the statement timeouts still apply.+ nil)))++(defn- with-probe-connection+ "Call `f` with an app-db connection whose reads are bounded by [[probe-network-timeout-ms]].+ The bound is lifted again before check-in: this is a pooled connection, and c3p0 hands the same physical one+ to whoever borrows it next, who is entitled to the app db's own patience -- a migration or a `copy-to-h2`+ would fail in seconds under the probe's. Both calls are local to the socket, so neither can hang.+ Check-in is therefore unbounded, which is the lesser problem: a connection whose read just timed out is+ broken, and c3p0 destroys rather than reuses it."+ [f]+ (with-open [conn (jdbc/get-connection (mdb/data-source))]+ (let [restore-to (network-timeout conn)]+ (set-network-timeout! conn probe-network-timeout-ms)+ (try+ (f conn)+ (finally+ (when restore-to+ (set-network-timeout! conn restore-to)))))))+(defn- app-db-can-provision-pgvector?"Whether the app-db user can create whichever store pieces are still missing, checked without persistingthem: the CREATEs run in a transaction that always rolls back.Attempts CREATE EXTENSION only when `create-extension?` and CREATE SCHEMA only when `create-schema?`, soan already-installed extension or existing schema needs no create privilege.- A privilege error reads as false."- [app-datasource create-extension? create-schema?]+ A refusal reads as false; a check that never got that far throws, so the caller can tell the two apart."+ [conn create-extension? create-schema?](try- (jdbc/with-transaction [tx app-datasource {:rollback-only true}]+ (jdbc/with-transaction [tx conn {:rollback-only true}](when create-extension?- (jdbc/execute! tx ["CREATE EXTENSION IF NOT EXISTS vector"]))+ (jdbc/execute! tx ["CREATE EXTENSION IF NOT EXISTS vector"]+ {:timeout probe-query-timeout-seconds}))(when create-schema?- (jdbc/execute! tx [(str "CREATE SCHEMA IF NOT EXISTS " (quoted/postgres app-db-schema))])))+ (jdbc/execute! tx [(str "CREATE SCHEMA IF NOT EXISTS " (quoted/postgres app-db-schema))]+ {:timeout probe-query-timeout-seconds})))true(catch Exception e+ (when-not (provisioning-denied? e)+ (throw e))(log/debugf "Semantic search: the application database user cannot provision the pgvector store: %s"(ex-message e))false)))@@ -255,27 +378,41 @@pg_available_extensions: managed Postgres often lists the extension as available while denying the DDL.The probe persists nothing, so the unlicensed and disabled instances whose availability predicates reachhere never mutate the app db; the persisted CREATE EXTENSION / CREATE SCHEMA run only on the activation- path ([[metabase-enterprise.semantic-search.pgvector-api/init-semantic-search!]])."+ path ([[metabase-enterprise.semantic-search.pgvector-api/init-semantic-search!]]).+ Runs on one [[with-probe-connection]], so a host that stops answering ends the check instead of stranding+ the readiness probe, which cannot be interrupted out of a stuck read."+ []+ (with-probe-connection+ (fn [conn]+ ;; The schema_exists SQL alias reads information_schema.schemata (privilege-filtered), not+ ;; pg_namespace. It answers "a semantic_search schema this role can use exists", not mere catalog+ ;; presence: a schema the app-db role lacks USAGE on reads as absent, so the store degrades to+ ;; unavailable rather than passing here and crashing later when init creates tables it can't write.+ (let [{:keys [installed available schema-exists]}+ (jdbc/execute-one! conn+ [(str "SELECT"+ " EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'vector') AS installed,"+ " EXISTS (SELECT 1 FROM pg_available_extensions WHERE name = 'vector') AS available,"+ " EXISTS (SELECT 1 FROM information_schema.schemata WHERE schema_name = ?) AS schema_exists")+ app-db-schema]+ {:builder-fn jdbc.rs/as-unqualified-kebab-maps+ :timeout probe-query-timeout-seconds})]+ (cond+ (not (or installed available)) false+ (and installed schema-exists) true+ :else (app-db-can-provision-pgvector? conn+ (not installed)+ (not schema-exists)))))))++(defn probe-app-db-store!+ "Whether the app db is a usable pgvector store right now: it answers, and it can still host the store.+ Uncached, so an extension dropped under a running instance shows up despite [[app-db-pgvector-support]]+ latching true.+ Asks [[check-app-db-pgvector-support]], the same question [[pgvector-mode]] selects `:app-db` on, so the+ two can't disagree. Demanding an installed extension instead would report every instance that has not+ activated semantic search yet -- where nothing has run `CREATE EXTENSION` -- as permanently unreachable."[]- (let [app-datasource (mdb/data-source)- ;; The schema_exists SQL alias reads information_schema.schemata (privilege-filtered), not- ;; pg_namespace. It answers "a semantic_search schema this role can use exists", not mere catalog- ;; presence: a schema the app-db role lacks USAGE on reads as absent, so the store degrades to- ;; unavailable rather than passing here and crashing later when init creates tables it can't write.- {:keys [installed available schema-exists]}- (jdbc/execute-one! app-datasource- [(str "SELECT"- " EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'vector') AS installed,"- " EXISTS (SELECT 1 FROM pg_available_extensions WHERE name = 'vector') AS available,"- " EXISTS (SELECT 1 FROM information_schema.schemata WHERE schema_name = ?) AS schema_exists")- app-db-schema]- {:builder-fn jdbc.rs/as-unqualified-kebab-maps})]- (cond- (not (or installed available)) false- (and installed schema-exists) true- :else (app-db-can-provision-pgvector? app-datasource- (not installed)- (not schema-exists)))))+ (check-app-db-pgvector-support))(defn- app-db-pgvector-supported?"Whether the application database can act as the pgvector store, via a cached probe.@@ -301,6 +438,9 @@" (MB_PGVECTOR_DB_URL is not set). Set it if this instance should use a"" dedicated pgvector database."))(reset! app-db-pgvector-support true)+ ;; Outcome before cooldown, always: a reader takes neither under the lock, and one that+ ;; saw the new cooldown beside the previous outcome would trust a result we just replaced.+ (reset! app-db-support-check-errored? false)(reset! probe-cooldown-timer nil)true)(do@@ -309,9 +449,11 @@" app-db user cannot create the vector extension or the semantic_search"" schema). Install pgvector and grant CREATE, or set MB_PGVECTOR_DB_URL;"" it is picked up automatically, no restart needed.")))+ (reset! app-db-support-check-errored? false)(reset! probe-cooldown-timer (u/start-timer))false))(catch Exception e+ (reset! app-db-support-check-errored? true)(reset! probe-cooldown-timer (u/start-timer))(log/warn (str "Semantic search: pgvector support check on the application database failed;"" will retry after the cooldown:")diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/db/store_health.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/db/store_health.cljnew file mode 100644index 00000000000..edfbe2f7fa5--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/db/store_health.clj@@ -0,0 +1,243 @@+(ns metabase-enterprise.semantic-search.db.store-health+ "Readiness of the pgvector store itself -- can this instance reach a vector database, and which one.+ The store is shared: semantic search and entity retrieval each provision their own tables in it, as more+ may later, so this reports the store rather than any one feature's index.+ Index health lives in [[metabase-enterprise.semantic-search.health]]."+ (:require+ [metabase-enterprise.semantic-search.db.datasource :as semantic.datasource]+ [metabase-enterprise.semantic-search.util :as semantic.u]+ [metabase.analytics-interface.core :as analytics]+ [metabase.analytics.core :as analytics.core]+ [metabase.app-db.core :as mdb]+ [metabase.health-inspector.core :as health-inspector]+ [metabase.search.index-health :as search.index-health]+ [metabase.util.log :as log])+ (:import+ (java.time Instant)))++(set! *warn-on-reflection* true)++(def ^:private storage-labels+ "The mutually exclusive pgvector backings, as `:storage` label values.+ A dedicated URL always wins over the app-db fallback, so at most one can be available at a time."+ ["dedicated" "appdb"])++(defonce ^:private ^{:doc "The last readiness probe, `{:storage :connected? :at}`, nil before the first.+ Shared with [[pgvector-store-health-check]] so it doesn't probe again."}+ last-readiness-probe+ (atom nil))++(def ^:private readiness-refresh-interval-seconds+ "How often the readiness probe runs.+ Hourly: an app db that can't host pgvector yet re-runs the support check, which may attempt rolled-back DDL."+ (* 60 60))++(def ^:private readiness-scrape-interval-seconds+ "How often the scrape path reconsiders probing.+ Far shorter than the probe interval, because the pull collector stamps its throttle before the refresh+ runs: an hourly slot spent on a probe that couldn't answer -- an app db still migrating at startup -- would+ leave the gauges wrong for the rest of the hour. The probe's own cadence is enforced by+ [[readiness-refresh-due?]] instead, off the last probe that did answer."+ 60)++(def ^:private probe-wait-ms+ "How long the health check waits on a probe before reporting without it.+ Above the JDBC timeouts that end one first: the dedicated probe's connect and socket timeouts, the app db's+ connection checkout plus statement timeouts.+ Only a probe none of those bound runs past it -- a blackholed host on a URL with no socketTimeout -- and+ holding the whole health-inspector run behind that would be worse than reporting the previous answer."+ (* 60 1000))++(defn- store-connected?+ "Whether `mode`'s store answers. A refused or failed connection is an answer, and reads as disconnected."+ [mode]+ (try+ (case mode+ :dedicated (do (semantic.datasource/probe-dedicated-connection!) true)+ :app-db (semantic.datasource/probe-app-db-store!)+ :unavailable false)+ (catch Exception e+ ;; The exception is the only record of why the gauge dropped.+ (log/warn e "Pgvector connection probe failed" {:mode mode})+ false)))++(defn- app-db-store-allowed?+ "Whether this instance may check the app db for pgvector support, the same boot-safe gate the semantic+ metric collector schedules on.+ Licensed, so an unlicensed instance never runs that check and its rolled-back DDL."+ []+ ;; TODO (Chris 2026-07-29) -- `:semantic-search` only.+ ;; Entity retrieval is dedicated-only today, so its instances never reach here; once it supports pgvector+ ;; on the app db, an app-db instance licensed for `:library-retrieval` alone would report no store at all.+ (semantic.u/semantic-search-configured?))++(def ^:private unresolved-store+ "What the probe reports when it couldn't find out: no store, and due to be retried on the next scrape+ rather than holding the hourly slot."+ {:mode :unavailable, :connected? false, :resolved? false})++(defn- resolve-store+ "Which store this instance uses, and whether it answers."+ []+ (cond+ ;; A dedicated URL always wins, and reading it asks the app db nothing.+ (semantic.datasource/dedicated-url-configured?)+ {:mode :dedicated, :connected? (store-connected? :dedicated), :resolved? true}++ ;; Everything below reads the license or probes the app db, and neither can answer until migrations+ ;; finish. Say so rather than guess, or the guess is what the gauges show for the next hour.+ (not (mdb/db-is-set-up?))+ unresolved-store++ (not (app-db-store-allowed?))+ {:mode :unavailable, :connected? false, :resolved? true}++ :else+ (let [mode (semantic.datasource/pgvector-mode)]+ {:mode mode+ :connected? (store-connected? mode)+ ;; A support check that errored also reads as :unavailable, and that is a guess, not an answer.+ :resolved? (or (not= mode :unavailable)+ (not @semantic.datasource/app-db-support-check-errored?))})))++(defn- probe-store+ "[[resolve-store]], reporting no store when it throws.+ Choosing the mode is itself a database call -- the app-db arm can attempt rolled-back DDL against a host+ that never answers -- so the whole resolution runs here, not only the connection probe."+ []+ (try+ (resolve-store)+ ;; Throwable, not Exception: an Error here would otherwise abandon the gauge writes, freezing every+ ;; series at its last value instead of dropping it.+ (catch Throwable e+ ;; The exception is the only record of why the gauges dropped.+ (log/warn e "Pgvector store probe failed")+ unresolved-store)))++(defn- clear-stale-last-success!+ "Drop the last-success timestamp when this instance switches from one backing to another.+ It is only ever written for the current storage, so the label left behind would otherwise sit at its final+ value for the life of the process, reading as a store that has not connected since.+ Losing the store is not a switch: that is precisely when the last known good timestamp is worth keeping."+ [previous-storage storage]+ (when (and previous-storage storage (not= previous-storage storage))+ ;; Clearing takes every label with it, which is what we want here -- only one can ever hold a value.+ (analytics/clear! :metabase-pgvector/store-last-success-timestamp-seconds)))++(defn- collect-pgvector-readiness-metrics!+ "Record availability and connection health for the selected pgvector store.+ Ignores engine activation, so a pgvector rollout can be validated before enabling anything that uses it.+ Needs no lock of its own: [[request-pgvector-readiness-refresh!]] admits one probe at a time."+ []+ (let [previous-storage (:storage @last-readiness-probe)+ {:keys [mode connected? resolved?]} (probe-store)+ storage (case mode :dedicated "dedicated" :app-db "appdb" nil)+ at (.getEpochSecond (Instant/now))]+ ;; Publish both stable series on every instance; both are zero when no store is usable.+ (doseq [candidate storage-labels]+ (analytics/set-gauge! :metabase-pgvector/store-available+ {:storage candidate}+ (if (= candidate storage) 1 0))+ (analytics/set-gauge! :metabase-pgvector/store-connected+ {:storage candidate}+ (if (and (= candidate storage) connected?) 1 0)))+ (clear-stale-last-success! previous-storage storage)+ (reset! last-readiness-probe {:storage storage+ :connected? connected?+ :resolved? resolved?+ :at at})+ (when connected?+ (analytics/set-gauge! :metabase-pgvector/store-last-success-timestamp-seconds+ {:storage storage}+ at))))++(defn- readiness-probe-stale?+ [{:keys [at]}]+ (or (nil? at)+ (> (- (.getEpochSecond (Instant/now)) ^long at)+ (* 2 readiness-refresh-interval-seconds))))++(defn- readiness-refresh-due?+ "Whether to probe again on this scrape.+ A probe that couldn't answer is retried on the next one; one that did answer holds for the full interval."+ [{:keys [at resolved?]}]+ (or (nil? at)+ (not resolved?)+ (>= (- (.getEpochSecond (Instant/now)) ^long at) readiness-refresh-interval-seconds)))++(defn- refresh-pgvector-readiness-metrics!+ []+ (try+ (collect-pgvector-readiness-metrics!)+ (catch Throwable e+ ;; The scrape path never derefs this, so a throw here is otherwise silent.+ (log/error e "Pgvector readiness metric refresh failed"))))++(defonce ^:private readiness-probe-future (atom nil))++(defn- submit-pgvector-readiness-refresh!+ [f]+ (future-call f))++(defn- request-pgvector-readiness-refresh!+ "Start a readiness probe unless one is still running, and return the running probe.+ A live probe is never replaced. One stuck in a socket read cannot be cancelled, so admitting a replacement+ would strand another thread on every scrape; leaving the gauges on their last values until it ends is the+ better of the two, and only a database that answers nothing at all gets there."+ []+ (locking readiness-probe-future+ (let [running @readiness-probe-future]+ (if (and running (not (realized? running)))+ running+ (reset! readiness-probe-future+ (submit-pgvector-readiness-refresh! refresh-pgvector-readiness-metrics!))))))++(defn- pgvector-store-health-check+ "Health-inspector row for pgvector store reachability, nil when there is no store to reach.+ Reports the last probe, waiting up to [[probe-wait-ms]] on a fresh one when that is stale, so an unscraped+ instance still gets a current row without holding up every other check behind an unresponsive database."+ []+ (when (readiness-probe-stale? @last-readiness-probe)+ (deref (request-pgvector-readiness-refresh!) probe-wait-ms nil))+ (let [{:keys [storage connected? resolved?] :as probe} @last-readiness-probe]+ (cond+ ;; nil is reserved for "this instance has no pgvector store", which is a fact. Not finding out is not,+ ;; whether the probe answered that it couldn't tell or is still out there when the wait runs out.+ (or (nil? probe) (not resolved?))+ (search.index-health/degraded "Could not determine whether a pgvector store is reachable.")++ (not storage) nil+ connected? (search.index-health/healthy (format "pgvector store (%s) reachable." storage))+ :else (search.index-health/degraded (format "pgvector store (%s) unreachable." storage)))))++(health-inspector/register-check! :pgvector-store pgvector-store-health-check)++;; Seed both series at startup so every process exposes them before its first probe.+(defmethod analytics.core/known-labels :metabase-pgvector/store-available [_]+ (for [storage storage-labels]+ {:storage storage}))++(defmethod analytics.core/known-labels :metabase-pgvector/store-connected [_]+ (analytics.core/known-labels :metabase-pgvector/store-available))++;; store-last-success-timestamp-seconds is deliberately absent: a seeded 0 reads as 1970, firing any+;; staleness alert forever.+;; It appears once a probe succeeds.++;; No initial-value either, though a configured dedicated URL is available without being probed:+;; observe-initial-values increments rather than sets, so seeding 1 would leave the gauge at 2 wherever a+;; scrape had already probed. Both series start at 0 and the first probe corrects them.++(defn- scrape-pgvector-readiness-gauges!+ "Probe when due, and never wait for the answer -- the scrape carries no database latency."+ []+ (when (readiness-refresh-due? @last-readiness-probe)+ (request-pgvector-readiness-refresh!))+ nil)++;; Prometheus scrapes every Metabase process, whereas a Quartz job runs on only one member of a cluster.+;; Start a local, single-flight background probe from the scrape path so every process refreshes its own+;; gauge series.+(defmethod analytics.core/pull-collector ::pgvector-readiness-gauges [_]+ {:min-interval-s readiness-scrape-interval-seconds+ :f scrape-pgvector-readiness-gauges!})diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/init.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/init.cljindex 61d95f9e33f..412f2788e63 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/init.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/init.clj@@ -1,5 +1,6 @@(ns metabase-enterprise.semantic-search.init(:require+ [metabase-enterprise.semantic-search.db.store-health][metabase-enterprise.semantic-search.embedding-health][metabase-enterprise.semantic-search.events][metabase-enterprise.semantic-search.health]diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/db/datasource_mode_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/db/datasource_mode_test.cljindex ef0b572bd0c..ee8b464b78a 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/db/datasource_mode_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/db/datasource_mode_test.clj@@ -10,10 +10,40 @@[metabase.test :as mt][next.jdbc :as jdbc])(:import- (clojure.lang ExceptionInfo)))+ (clojure.lang ExceptionInfo)+ (java.sql Connection)))(set! *warn-on-reflection* true)+(defn- stub-connection+ "The support check borrows a connection to bound it at the socket level; these tests stub every statement+ on it, so it only has to close, report its timeout, and record the ones it is handed."+ (^Connection [] (stub-connection 0 (atom [])))+ (^Connection [initial-ms timeouts]+ (reify Connection+ (getNetworkTimeout [_] initial-ms)+ (setNetworkTimeout [_ _ ms] (swap! timeouts conj ms))+ (close [_] nil))))++(deftest probe-connection-restores-its-timeout-test+ (testing "the probe's socket bound is lifted before check-in, so the next borrower keeps the app db's own"+ (let [with-probe-connection @#'semantic.db.datasource/with-probe-connection+ probe-bound @#'semantic.db.datasource/probe-network-timeout-ms]+ (doseq [initial-ms [0 300000]]+ (testing (format "from an existing timeout of %dms" initial-ms)+ (let [timeouts (atom [])]+ (with-redefs [mdb/data-source (constantly ::app-pool)+ jdbc/get-connection (fn [_] (stub-connection initial-ms timeouts))]+ (is (= ::probed (with-probe-connection (constantly ::probed))))+ (is (= [probe-bound initial-ms] @timeouts))))))+ (testing "and when the probe throws, which is how a stuck read ends"+ (let [timeouts (atom [])]+ (with-redefs [mdb/data-source (constantly ::app-pool)+ jdbc/get-connection (fn [_] (stub-connection 300000 timeouts))]+ (is (thrown? ExceptionInfo+ (with-probe-connection (fn [_] (throw (ex-info "stuck" {}))))))+ (is (= [probe-bound 300000] @timeouts))))))))+(defmacro ^:private with-support-cache"Run body with all three pieces of app-db probe state rebound to fresh atoms: the support cache (holding`init`), the cooldown timer, and the log-once hint latch."@@ -57,7 +87,8 @@(testing "supported only when the vector extension and the semantic_search schema exist or can be created"(letfn [(check [] (semantic.db.datasource/check-app-db-pgvector-support))(catalog [m] (fn [& _] m))]- (with-redefs [mdb/data-source (constantly ::app-pool)]+ (with-redefs [mdb/data-source (constantly ::app-pool)+ jdbc/get-connection (fn [_] (stub-connection))](testing "extension neither installed nor available → unsupported, no provisioning probe"(with-redefs [jdbc/execute-one! (catalog {:installed false :available false :schema-exists false})semantic.db.datasource/app-db-can-provision-pgvector?diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/db/store_health_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/db/store_health_test.cljnew file mode 100644index 00000000000..02d9fac3206--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/db/store_health_test.clj@@ -0,0 +1,284 @@+(ns metabase-enterprise.semantic-search.db.store-health-test+ (:require+ [clojure.string :as str]+ [clojure.test :refer [deftest is testing]]+ [iapetos.export :as export]+ [java-time.api :as t]+ [metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource]+ [metabase-enterprise.semantic-search.db.store-health :as semantic.store-health]+ [metabase-enterprise.semantic-search.util :as semantic.u]+ [metabase.analytics-interface.core :as analytics]+ [metabase.analytics.core :as analytics.core]+ [metabase.app-db.core :as mdb]+ [metabase.test :as mt]))++(defn- readiness-gauges+ "Every readiness gauge for both storage labels, as `{storage {:available _ :connected _ :last-success _}}`."+ [system]+ (into {}+ (for [storage ["dedicated" "appdb"]]+ [storage {:available (mt/metric-value system :metabase-pgvector/store-available+ {:storage storage})+ :connected (mt/metric-value system :metabase-pgvector/store-connected+ {:storage storage})+ :last-success (mt/metric-value+ system :metabase-pgvector/store-last-success-timestamp-seconds+ {:storage storage})}])))++(deftest pgvector-readiness-metrics-test+ (mt/with-prometheus-system! [_ system]+ (testing "an instance without any pgvector backend is explicitly not ready"+ (mt/with-dynamic-fn-redefs+ [semantic.db.datasource/dedicated-url-configured? (constantly false)+ semantic.db.datasource/pgvector-mode (constantly :unavailable)+ semantic.db.datasource/probe-dedicated-connection! #(throw (ex-info "should not connect" {}))]+ (@#'semantic.store-health/collect-pgvector-readiness-metrics!)+ (is (=? {"dedicated" {:available zero?, :connected zero?}+ "appdb" {:available zero?, :connected zero?}}+ (readiness-gauges system)))))+ (testing "a connected dedicated store is available and healthy"+ (mt/with-dynamic-fn-redefs+ [semantic.db.datasource/dedicated-url-configured? (constantly true)+ semantic.db.datasource/probe-dedicated-connection! (constantly {:one 1})]+ (@#'semantic.store-health/collect-pgvector-readiness-metrics!))+ (is (=? {"dedicated" {:available #(== 1 %), :connected #(== 1 %), :last-success pos?}+ "appdb" {:available zero?, :connected zero?}}+ (readiness-gauges system)))+ (let [last-success (get-in (readiness-gauges system) ["dedicated" :last-success])]+ (testing "a later failure clears connected but preserves the last-success timestamp"+ (mt/with-dynamic-fn-redefs+ [semantic.db.datasource/dedicated-url-configured? (constantly true)+ semantic.db.datasource/probe-dedicated-connection! #(throw (ex-info "probe failed" {}))]+ (@#'semantic.store-health/collect-pgvector-readiness-metrics!))+ (is (=? {"dedicated" {:available #(== 1 %), :connected zero?, :last-success #(== last-success %)}}+ (readiness-gauges system))))))+ (testing "a connected app-db pgvector store is available and healthy"+ (mt/with-dynamic-fn-redefs+ [mdb/db-is-set-up? (constantly true)+ semantic.db.datasource/dedicated-url-configured? (constantly false)+ semantic.u/semantic-search-configured? (constantly true)+ semantic.db.datasource/pgvector-mode (constantly :app-db)+ semantic.db.datasource/probe-app-db-store! (constantly true)]+ (@#'semantic.store-health/collect-pgvector-readiness-metrics!))+ (is (=? {"dedicated" {:available zero?, :connected zero?}+ "appdb" {:available #(== 1 %), :connected #(== 1 %), :last-success pos?}}+ (readiness-gauges system))))+ (testing "an app-db store whose vector extension went away reads available but disconnected"+ (mt/with-dynamic-fn-redefs+ [mdb/db-is-set-up? (constantly true)+ semantic.db.datasource/dedicated-url-configured? (constantly false)+ semantic.u/semantic-search-configured? (constantly true)+ semantic.db.datasource/pgvector-mode (constantly :app-db)+ semantic.db.datasource/probe-app-db-store! (constantly false)]+ (@#'semantic.store-health/collect-pgvector-readiness-metrics!))+ (is (=? {"appdb" {:available #(== 1 %), :connected zero?}}+ (readiness-gauges system))))))++(def ^:private last-success-sample-re+ #"^metabase_pgvector_store_last_success_timestamp_seconds\{storage=\"([^\"]+)\",?\} (\S+)")++(defn- exposed-last-success+ "The last-success samples a scrape would actually see, as `{storage value}`.+ Read from the exposition rather than [[mt/metric-value]], which creates the child it looks up and so+ reports an absent series as a present zero."+ [system]+ (into {}+ (keep (fn [line]+ (when-let [[_ storage value] (re-find last-success-sample-re line)]+ [storage (parse-double value)])))+ (str/split-lines (export/text-format (:registry system)))))++(deftest last-success-does-not-outlive-its-storage-test+ (mt/with-prometheus-system! [_ system]+ ;; Scraping runs the pull collectors, this namespace's included. Stub the submit for the whole test so a+ ;; read of the exposition can't spawn a real probe that races it by writing the series being read.+ (mt/with-dynamic-fn-redefs+ [semantic.store-health/submit-pgvector-readiness-refresh! (constantly nil)]+ (mt/with-dynamic-fn-redefs+ [semantic.db.datasource/dedicated-url-configured? (constantly true)+ semantic.db.datasource/probe-dedicated-connection! (constantly {:one 1})]+ (@#'semantic.store-health/collect-pgvector-readiness-metrics!))+ (is (=? {"dedicated" pos?} (exposed-last-success system)))+ (testing "switching storage drops the old timestamp, which would otherwise never advance again"+ (mt/with-dynamic-fn-redefs+ [mdb/db-is-set-up? (constantly true)+ semantic.db.datasource/dedicated-url-configured? (constantly false)+ semantic.u/semantic-search-configured? (constantly true)+ semantic.db.datasource/pgvector-mode (constantly :app-db)+ semantic.db.datasource/probe-app-db-store! (constantly true)]+ (@#'semantic.store-health/collect-pgvector-readiness-metrics!))+ (is (=? {"appdb" pos?} (exposed-last-success system)))+ (is (= #{"appdb"} (set (keys (exposed-last-success system))))+ "the abandoned label is gone rather than zeroed -- a 0 would read as 1970"))+ (testing "losing the store keeps the timestamp: that is when it is worth knowing"+ (mt/with-dynamic-fn-redefs+ [mdb/db-is-set-up? (constantly true)+ semantic.db.datasource/dedicated-url-configured? (constantly false)+ semantic.u/semantic-search-configured? (constantly false)]+ (@#'semantic.store-health/collect-pgvector-readiness-metrics!))+ (is (=? {"appdb" pos?} (exposed-last-success system)))))))++(deftest probe-store-test+ (let [probe @#'semantic.store-health/probe-store]+ (mt/with-dynamic-fn-redefs [semantic.db.datasource/dedicated-url-configured? (constantly true)]+ (testing "an ordinary failure reads as disconnected, but the store is still known"+ (mt/with-dynamic-fn-redefs+ [semantic.db.datasource/probe-dedicated-connection! #(throw (ex-info "nope" {}))]+ (is (=? {:mode :dedicated, :connected? false, :resolved? true} (probe)))))+ (testing "an Error does not escape and abandon the gauge writes"+ (mt/with-dynamic-fn-redefs+ [semantic.db.datasource/probe-dedicated-connection! #(throw (AssertionError. "boom"))]+ (is (=? {:mode :unavailable, :connected? false, :resolved? false} (probe)))))+ (testing "a store that answers reads as connected"+ (mt/with-dynamic-fn-redefs+ [semantic.db.datasource/probe-dedicated-connection! (constantly {:one 1})]+ (is (=? {:mode :dedicated, :connected? true, :resolved? true} (probe))))))+ (testing "a support check that errored is unresolved: unsupported and unknown both read as :unavailable"+ (reset! semantic.db.datasource/app-db-support-check-errored? true)+ (try+ (mt/with-dynamic-fn-redefs+ [mdb/db-is-set-up? (constantly true)+ semantic.db.datasource/dedicated-url-configured? (constantly false)+ semantic.u/semantic-search-configured? (constantly true)+ semantic.db.datasource/pgvector-mode (constantly :unavailable)]+ (is (=? {:mode :unavailable, :connected? false, :resolved? false} (probe)))+ (testing "one that answered no is an answer, and holds for the interval"+ (reset! semantic.db.datasource/app-db-support-check-errored? false)+ (is (=? {:mode :unavailable, :connected? false, :resolved? true} (probe)))))+ (finally+ (reset! semantic.db.datasource/app-db-support-check-errored? false))))+ (testing "an app db that is not migrated yet is unresolved, not an answer to cache for an hour"+ (mt/with-dynamic-fn-redefs+ [mdb/db-is-set-up? (constantly false)+ semantic.db.datasource/dedicated-url-configured? (constantly false)+ semantic.db.datasource/pgvector-mode #(throw (ex-info "must not probe" {}))]+ (is (=? {:mode :unavailable, :connected? false, :resolved? false} (probe)))))))++(deftest ^:sequential live-probe-is-never-replaced-test+ (testing "a probe stuck on an unresponsive database cannot be cancelled, so no replacement is admitted"+ (let [probe-future @#'semantic.store-health/readiness-probe-future+ request @#'semantic.store-health/request-pgvector-readiness-refresh!+ started (promise)+ release (promise)+ starts (atom 0)]+ (try+ (reset! probe-future nil)+ (mt/with-dynamic-fn-redefs+ [semantic.store-health/refresh-pgvector-readiness-metrics!+ ;; Counted before it is announced, so awaiting `started` proves the count is in.+ (fn [] (swap! starts inc) (deliver started true) (deref release 10000 ::hung))]+ (let [first-probe (request)]+ (is (identical? first-probe (request)) "the running probe is returned rather than replaced")+ ;; The request only submits, so wait for the body rather than racing the scheduler.+ (is (true? (deref started 10000 ::never-ran)))+ (is (= 1 @starts) "and no second one is started")+ (deliver release true)+ (is (true? (deref first-probe 10000 ::hung)))+ (testing "once it ends, the next request starts a fresh one"+ (let [next-probe (request)]+ (is (not (identical? first-probe next-probe)))+ (is (true? (deref next-probe 10000 ::hung)))+ (is (= 2 @starts))))))+ (finally+ (deliver release true)+ (reset! probe-future nil))))))++(deftest readiness-refresh-due-test+ (let [due? @#'semantic.store-health/readiness-refresh-due?+ now (quot (t/to-millis-from-epoch (t/instant)) 1000)]+ (is (true? (due? nil)) "nothing probed yet")+ (is (false? (due? {:at now, :resolved? true})) "a fresh answer holds the interval")+ (is (true? (due? {:at now, :resolved? false}))+ "a probe that could not answer is retried rather than holding the hourly slot")+ (is (true? (due? {:at (- now 3601), :resolved? true})) "the interval has passed")))++(deftest pgvector-readiness-skips-unlicensed-app-db-probe-test+ (testing "an unlicensed instance never resolves pgvector-mode, whose app-db arm probes the app db"+ (mt/with-premium-features #{}+ (mt/with-prometheus-system! [_ system]+ (mt/with-dynamic-fn-redefs+ [mdb/db-is-set-up? (constantly true)+ semantic.db.datasource/dedicated-url-configured? (constantly false)+ semantic.db.datasource/pgvector-mode #(throw (ex-info "must not probe" {}))]+ (@#'semantic.store-health/collect-pgvector-readiness-metrics!)+ (is (=? {"dedicated" {:available zero?, :connected zero?}+ "appdb" {:available zero?, :connected zero?}}+ (readiness-gauges system))))))))++(deftest initial-values-test+ (testing "both storage series are seeded at startup, and the timestamp deliberately is not"+ (is (= [{:storage "dedicated"} {:storage "appdb"}]+ (analytics.core/known-labels :metabase-pgvector/store-available)+ (analytics.core/known-labels :metabase-pgvector/store-connected)))+ (is (not (contains? (methods analytics.core/known-labels)+ :metabase-pgvector/store-last-success-timestamp-seconds))+ "a seeded 0 would read as 1970 and fire any staleness alert forever"))+ (testing "every series is seeded at 0, even one a configured dedicated URL already makes available"+ (mt/with-dynamic-fn-redefs [semantic.db.datasource/dedicated-url-configured? (constantly true)]+ (doseq [storage ["dedicated" "appdb"]]+ (is (zero? (analytics.core/initial-value :metabase-pgvector/store-available {:storage storage}))+ "observe-initial-values increments, so a non-zero seed doubles a gauge a scrape already set")))))++(defn- fresh-probe+ [m]+ (assoc m :at (quot (t/to-millis-from-epoch (t/instant)) 1000)))++(deftest pgvector-store-health-check-test+ (let [check @#'semantic.store-health/pgvector-store-health-check+ probe @#'semantic.store-health/last-readiness-probe+ collects (atom 0)]+ (mt/with-dynamic-fn-redefs+ [semantic.store-health/collect-pgvector-readiness-metrics! #(swap! collects inc)]+ (testing "a fresh probe is reported as-is, without running another"+ (reset! probe (fresh-probe {:storage nil, :connected? false, :resolved? true}))+ (is (nil? (check)) "omitted when there is no store to reach")+ (reset! probe (fresh-probe {:storage "dedicated", :connected? true, :resolved? true}))+ (is (=? {:health 100, :message #"pgvector store \(dedicated\) reachable\."} (check)))+ (reset! probe (fresh-probe {:storage "appdb", :connected? false, :resolved? true}))+ (is (=? {:health 0, :message #"pgvector store \(appdb\) unreachable\."} (check)))+ (is (zero? @collects)))+ (testing "a probe that could not answer is degraded, not omitted as inapplicable"+ (reset! probe (fresh-probe {:storage nil, :connected? false, :resolved? false}))+ (is (=? {:health 0, :message #"Could not determine whether a pgvector store is reachable\."} (check))))+ (testing "a missing or stale probe is refreshed, so an unscraped instance still reports"+ (reset! probe nil)+ ;; The refresh is stubbed, so the atom is still empty when the wait ends -- the case where a probe+ ;; is still out there, which must not read as an absent store.+ (is (=? {:health 0, :message #"Could not determine whether a pgvector store is reachable\."} (check)))+ (reset! probe {:storage "dedicated", :connected? true, :resolved? true, :at 0})+ (check)+ (is (= 2 @collects))))))++(deftest ^:sequential pull-collector-refreshes-pgvector-metrics-on-each-instance-test+ (let [probe-future @#'semantic.store-health/readiness-probe-future+ probe @#'semantic.store-health/last-readiness-probe+ prior-probe @probe+ collector (analytics.core/pull-collector ::semantic.store-health/pgvector-readiness-gauges)+ gauge-calls (atom [])+ refreshes (atom 0)+ submitted (atom [])+ ;; The submitted probe has to stay unrealized, or the second scrape sees a finished one and starts+ ;; its own -- which is the single-flight behaviour under test.+ hold (promise)]+ (try+ (reset! probe-future nil)+ ;; Nothing probed yet, so the refresh is due.+ (reset! probe nil)+ (mt/with-dynamic-fn-redefs+ [analytics/set-gauge! #(swap! gauge-calls conj (vec %&))+ semantic.store-health/collect-pgvector-readiness-metrics! #(swap! refreshes inc)+ semantic.store-health/submit-pgvector-readiness-refresh! #(do (swap! submitted conj %)+ (future (deref hold 10000 ::hung)))]+ ((:f collector))+ ((:f collector))+ (is (= 1 (count @submitted)) "overlapping scrapes share one local refresh")+ (is (empty? @gauge-calls) "the scrape path writes no gauges of its own")+ (is (zero? @refreshes) "the database probe does not run on the synchronous scrape path")+ ((first @submitted)))+ (is (= 60 (:min-interval-s collector))+ "the scrape reconsiders often; the hourly cadence is readiness-refresh-due?'s job")+ (is (= 1 @refreshes))+ (finally+ (deliver hold true)+ (reset! probe-future nil)+ (reset! probe prior-probe)))))diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/db_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/db_test.cljindex c9ecaa76be6..fe771274cdd 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/db_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/db_test.clj@@ -3,7 +3,9 @@[clojure.test :refer :all][metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource])(:import- (com.mchange.v2.c3p0 PoolBackedDataSource)))+ (com.mchange.v2.c3p0 PoolBackedDataSource)+ (java.sql SQLException SQLTimeoutException)+ (javax.sql DataSource)))(set! *warn-on-reflection* true)@@ -49,10 +51,35 @@(finally(reset! semantic.db.datasource/data-source orig-data-source))))))+(deftest probe-dedicated-connection-test+ (testing "the readiness probe connects without initializing the dedicated pool"+ (when semantic.db.datasource/db-url+ (with-redefs [semantic.db.datasource/data-source (atom nil)]+ (is (= {:test 1} (semantic.db.datasource/probe-dedicated-connection!)))+ (is (nil? @semantic.db.datasource/data-source))))))+(def ^:private parse-db-url #'semantic.db.datasource/parse-db-url)+(def ^:private probe-jdbc-url #'semantic.db.datasource/probe-jdbc-url)+(def ^:private base-url "jdbc:postgresql://localhost:5432/mb_semantic_search")+(deftest probe-jdbc-url-test+ (testing "the probe URL fills in the fail-fast timeouts the one-shot datasource would otherwise lack"+ ;; db-url is a value, not a fn, so with-dynamic-fn-redefs can't bind it+ (with-redefs [semantic.db.datasource/db-url base-url]+ (is (= (str base-url "?connectTimeout=5&socketTimeout=10")+ (probe-jdbc-url)))))+ (testing "a timeout the operator set on MB_PGVECTOR_DB_URL is replaced, not deferred to"+ ;; An abandoned probe can't be interrupted out of a socket read, so it must not inherit a long wait.+ (with-redefs [semantic.db.datasource/db-url (str base-url "?socketTimeout=60")]+ (is (= (str base-url "?connectTimeout=5&socketTimeout=10")+ (probe-jdbc-url)))))+ (testing "unrelated connection params pass through, in the order they were written"+ (with-redefs [semantic.db.datasource/db-url (str base-url "?user=mb&sslmode=require")]+ (is (= (str base-url "?user=mb&sslmode=require&connectTimeout=5&socketTimeout=10")+ (probe-jdbc-url))))))+(deftest parse-db-url-defaults-test(testing "a URL with no params leaves the URL untouched and uses the default pool props";; strict = (not =?) so an unexpected extra/missing prop also fails, not just a wrong value@@ -142,3 +169,38 @@(is (= "metabase-semantic-search-db" (.getDataSourceName pool-ds))))(finally(semantic.db.datasource/shutdown-db!))))))))++(def ^:private can-provision? #'semantic.db.datasource/app-db-can-provision-pgvector?)++(defn- failing-datasource+ "A datasource whose every connection attempt throws `e`, so the provisioning check fails before it can+ ask the question."+ ^DataSource [^Exception e]+ (reify DataSource+ (getConnection [_] (throw e))+ (getConnection [_ _ _] (throw e))))++(deftest app-db-can-provision-pgvector-distinguishes-refusal-from-failure-test+ (testing "the database refusing to provision is an answer, and reads as no"+ (doseq [[state what] {"42501" "the role lacks CREATE"+ "0A000" "the extension is not available on this server"+ "58P01" "the extension's control file is missing"}]+ (testing what+ ;; Hinted: SQLException also has a (String, Throwable) ctor, so untyped args reflect.+ (is (false? (can-provision? (failing-datasource (SQLException. ^String what ^String state))+ true true))))))+ (testing "a check that never got an answer throws, rather than reading as a settled no"+ (doseq [[e what] {(SQLTimeoutException. "statement timeout" "57014") "a timed-out DDL probe"+ (SQLException. "connection refused" "08006") "a dropped connection"+ (SQLException. "too many connections" "53300") "an exhausted server"+ (InterruptedException. "abandoned") "an interrupted probe"}]+ (testing what+ (is (thrown? Exception (can-provision? (failing-datasource e) true true))))))+ (testing "a refusal wrapped in another exception is still recognised"+ (let [wrapped (SQLException. "rollback failed" "25P02" (SQLException. "denied" "42501"))]+ (is (false? (can-provision? (failing-datasource wrapped) true true)))))+ (testing "a refusal hung off getNextException is still recognised"+ ;; JDBC chains sibling errors there rather than through getCause, so a cause-only walk misses it.+ (let [chained (doto (SQLException. "batch failed" "25P02")+ (.setNextException (SQLException. "denied" "42501")))]+ (is (false? (can-provision? (failing-datasource chained) true true))))))diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/task/metric_collector_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/task/metric_collector_test.cljindex a5cf3401da5..21542f73510 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/task/metric_collector_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/task/metric_collector_test.clj@@ -70,8 +70,8 @@(deftest shared-index-metrics-survive-semantic-collector-failure-test(let [refreshes (atom 0)](mt/with-dynamic-fn-redefs- [semantic.u/semantic-search-active? (constantly true)- semantic.env/get-pgvector-datasource! #(throw (ex-info "pgvector unavailable" {}))+ [semantic.u/semantic-search-active? (constantly true)+ semantic.env/get-pgvector-datasource! #(throw (ex-info "pgvector unavailable" {}))search.index-health/refresh-search-index-metrics! #(swap! refreshes inc)](@#'semantic.task.collector/collect-metrics!)(is (= 1 @refreshes)))))@@ -79,8 +79,8 @@(deftest interrupted-semantic-collector-skips-shared-refresh-test(let [refreshes (atom 0)](mt/with-dynamic-fn-redefs- [semantic.u/semantic-search-active? (constantly true)- semantic.env/get-pgvector-datasource! #(throw (InterruptedException.))+ [semantic.u/semantic-search-active? (constantly true)+ semantic.env/get-pgvector-datasource! #(throw (InterruptedException.))search.index-health/refresh-search-index-metrics! #(swap! refreshes inc)](is (thrown? InterruptedException (@#'semantic.task.collector/collect-metrics!)))(is (zero? @refreshes)))))@@ -91,13 +91,14 @@(let [pgvector (semantic.env/get-pgvector-datasource!)index-metadata (semantic.tu/unique-index-metadata)model semantic.tu/mock-embedding-model]- (mt/with-dynamic-fn-redefs [semantic.env/get-index-metadata (fn [] index-metadata)- semantic.env/get-configured-embedding-model (fn [] model)- ;; supported? requires a configured embedder for engine selection- semantic.embedding/get-configured-model (fn [] model)- ;; collect-metrics! only runs when semantic is the active engine; pin it so a- ;; sibling test leaking the search-engine setting can't make this read 0- semantic.u/semantic-search-active? (fn [] true)]+ (mt/with-dynamic-fn-redefs+ [semantic.env/get-index-metadata (constantly index-metadata)+ semantic.env/get-configured-embedding-model (constantly model)+ ;; supported? requires a configured embedder for engine selection+ semantic.embedding/get-configured-model (constantly model)+ ;; collect-metrics! only runs when semantic is the active engine; pin it so a sibling test+ ;; leaking the search-engine setting can't make this read 0+ semantic.u/semantic-search-active? (constantly true)](testing "Missing tables are handled gracefully"(let [result (try(@#'semantic.task.collector/collect-metrics!)diff --git a/src/metabase/analytics/prometheus.clj b/src/metabase/analytics/prometheus.cljindex b6a3390ea6e..6b438c72706 100644--- a/src/metabase/analytics/prometheus.clj+++ b/src/metabase/analytics/prometheus.clj@@ -331,6 +331,21 @@(prometheus/counter :metabase-remote-sync/git-operations-failed{:description "Number of failed git operations":labels [:operation :remote]})+ ;; Shared: semantic search and entity retrieval each provision their own tables in it, as more may later.+ ;; At most one is available at a time: a dedicated MB_PGVECTOR_DB_URL always wins over the app db.+ (prometheus/gauge :metabase-pgvector/store-available+ {:description "Whether the given pgvector storage is available to this instance."+ :labels [:storage]})+ ;; Probed hourly, so an outage takes up to that long to show up. The dedicated probe opens its own+ ;; connection, so a store that is up but whose pool is saturated still reads as connected --+ ;; metabase_database_c3p0_* covers that. The app-db probe shares the application pool, and does not.+ (prometheus/gauge :metabase-pgvector/store-connected+ {:description "Whether the last connection probe to the given pgvector storage succeeded."+ :labels [:storage]})+ ;; Absent until a probe succeeds, and reset when the instance changes which backing it uses.+ (prometheus/gauge :metabase-pgvector/store-last-success-timestamp-seconds+ {:description "Unix timestamp of the last successful probe to the given pgvector storage."+ :labels [:storage]})(prometheus/counter :metabase-search/index-reindexes{:description "Number of reindexed search entries":labels [:model]})