The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Add operator knobs for the pgvector connection pool (#75651)
Chris Truterb934ac131de6
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/db/datasource.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/db/datasource.cljindex b6d7872fc92..b299d7de5bc 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/db/datasource.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/db/datasource.clj@@ -1,11 +1,16 @@(ns metabase-enterprise.semantic-search.db.datasource(:require+ [clojure.string :as str][environ.core :refer [env]][metabase.connection-pool :as connection-pool]+ [metabase.util :as u][metabase.util.log :as log][next.jdbc :as jdbc])(:import- (com.mchange.v2.c3p0 DataSources)))+ (com.mchange.v2.c3p0 DataSources PooledDataSource)+ (java.net URLDecoder)+ (java.nio.charset StandardCharsets)+ (org.postgresql PGProperty)))(set! *warn-on-reflection* true)@@ -17,37 +22,140 @@"The database URL used to connect to pgvector"(env :mb-pgvector-db-url))-;; See metabase.app-db.connection-pool-setup for more details on these properties-;; TODO: not sure if we need the MetabaseConnectionCustomizer like in the app DB connection pool setup-(def ^:private semantic-search-connection-pool-props- "Connection pool properties for semantic search database using c3p0."+;; All pgvector config — connection *and* connection-pool parameters — rides MB_PGVECTOR_DB_URL, so a+;; deployment sets everything in one place (the DB secret) with no env-var-vs-URL precedence to reason+;; about. c3p0 pool params can't ride the JDBC URL into the driver, so we parse them out here (see+;; parse-db-url) and hand the rest to pgjdbc. pgjdbc *silently ignores* params it doesn't recognize, so we+;; validate every param against an allowlist and throw on startup rather than let a typo become a+;; silently-dropped setting. Cf. metabase.app-db.connection-pool-setup for the analogous app-db pool.++;; c3p0 props we always set and don't expose for tuning.+(def ^:private fixed-pool-props{"idleConnectionTestPeriod" 60- "maxIdleTimeExcessConnections" (* 10 60) ; 5 minutes- "maxConnectionAge" (* 30 60) ; 30 minutes- "maxPoolSize" 5 ; Small pool to start- "minPoolSize" 1- "initialPoolSize" 1+ "maxIdleTimeExcessConnections" (* 10 60) ; cull excess idle connections after 10 minutes+ "maxConnectionAge" (* 30 60) ; recycle any connection after 30 minutes+ ;; c3p0 defaults to 3 per miss; 1 avoids tripling the cold-start burst on the shared RDS.+ "acquireIncrement" 1"dataSourceName" "metabase-semantic-search-db"})-(defn- build-db-config- "Build database configuration from environment variables"+(defn- parse-bool [^String s]+ (case (u/lower-case-en (str/trim s)) "true" true "false" false nil))++;; Overridable c3p0 knobs: name -> [default parse-fn]. Operators override by setting the same key on+;; MB_PGVECTOR_DB_URL; names match c3p0's own so they map straight onto its docs. parse-fn returns nil on a+;; malformed value, which we surface as a startup error.+(def ^:private tunable-pool-props+ {;; 0 idle connections is what makes a shared RDS viable. Cold start costs one handshake, warmed+ ;; concurrently with the embedding round-trip (see semantic-search.index/query-index). Set minPoolSize=1+ ;; to pin a warm connection on a hot instance.+ "minPoolSize" [0 parse-long]+ "initialPoolSize" [0 parse-long]+ "maxPoolSize" [5 parse-long]+ ;; Fail fast on pool exhaustion (c3p0 default 0 blocks forever); the engine falls back to appdb search+ ;; on error (see semantic-search.core/results).+ "checkoutTimeout" [10000 parse-long]+ ;; c3p0's leak detector, off by default: it destroys connections by checkout *duration*, so any value+ ;; low enough to catch leaks also kills slow work (reindex DDL, bulk upserts). Enable with the+ ;; stack-traces knob on a canary to hunt a suspected leak.+ "unreturnedConnectionTimeout" [0 parse-long]+ "debugUnreturnedConnectionStackTraces" [false parse-bool]+ ;; Off by default (idleConnectionTestPeriod covers idle conns); matches app-db. Flip on if the shared+ ;; RDS drops connections under the pool.+ "testConnectionOnCheckout" [false parse-bool]})++;; Postgres connection properties pgjdbc recognizes on the classpath; read at runtime so it tracks the+;; driver version. Anything on the URL that's neither a tunable pool knob nor one of these is a typo.+;; NB: explicit fn rather than the 1.12 `PGProperty/.getName` method value — eastwood's analyzer can't+;; parse the latter yet.+(def ^:private known-connection-params+ (into #{} (map #(.getName ^PGProperty %)) (PGProperty/values)))++(defn- url-decode ^String [^String s]+ (URLDecoder/decode s StandardCharsets/UTF_8))++(defn- split-query+ "Split a JDBC URL into [base pairs], where pairs is a seq of raw [key value] strings.+ Pairs are left verbatim so connection params pass through to pgjdbc exactly as written; only pool-knob+ values are decoded, in parse-db-url."+ [^String url]+ (let [[base query] (str/split url #"\?" 2)]+ ;; split each pair on its first '=' only, so a value may itself contain '=' (e.g. options=-c foo=bar)+ [base (for [pair (some-> query (str/split #"&"))+ :when (seq pair)+ :let [[k v] (str/split pair #"=" 2)]]+ [k (or v "")])]))++(defn- parse-db-url+ "Parse a pgvector JDBC URL into {:jdbc-url ... :pool-props ...}, or throw if it carries an unrecognized+ parameter."+ [^String url]+ (let [[base pairs] (split-query url)+ default-pool (merge fixed-pool-props+ (update-vals tunable-pool-props first))+ {:keys [pool conn]}+ (reduce+ (fn [acc [k raw-v]]+ (cond+ ;; a c3p0 knob: coerce and apply to the pool (pgjdbc can't read these off the URL)+ (contains? tunable-pool-props k)+ (let [parse (second (tunable-pool-props k))+ parsed (parse (url-decode raw-v))]+ (when (nil? parsed)+ (throw (ex-info (format "Invalid value for pgvector pool parameter %s: %s" k raw-v)+ {:param k, :value raw-v})))+ (assoc-in acc [:pool k] parsed))++ ;; a recognized pgjdbc connection property: leave it on the URL for the driver+ (contains? known-connection-params k)+ (update acc :conn conj (str k "=" raw-v))++ ;; pgjdbc would silently ignore anything else, so treat it as a typo and fail loudly+ :else+ (throw (ex-info (format (str "Unknown pgvector URL parameter %s. Expected a c3p0 pool knob "+ "(%s) or a Postgres connection property.")+ k (str/join ", " (sort (keys tunable-pool-props))))+ {:param k}))))+ {:pool default-pool, :conn []}+ pairs)]+ {:jdbc-url (cond-> base (seq conn) (str "?" (str/join "&" conn)))+ :pool-props pool}))++(defn- parsed-db-config+ "Parse [[db-url]] into {:jdbc-url ... :pool-props ...}, or throw if it is unset."[](if db-url- {:jdbcUrl db-url}+ (parse-db-url db-url)(throw (ex-info "MB_PGVECTOR_DB_URL environment variable is required for semantic search" {}))))+(defn shutdown-db!+ "Close the pooled data source (if any) and clear it, releasing all server-side connections.+ Safe to call when the pool was never initialized; a later [[init-db!]] builds a fresh pool."+ []+ (locking data-source+ (when-let [ds @data-source]+ (reset! data-source nil)+ (log/info "Closing semantic search connection pool")+ (.close ^PooledDataSource ds))))++;; Tests redef [[data-source]] and close their pools themselves; this hook only sees the root binding,+;; so it closes at most the production pool.+(defonce ^:private shutdown-hook+ (delay (.addShutdownHook (Runtime/getRuntime)+ (Thread. ^Runnable shutdown-db! "semantic-search-pool-shutdown"))))+(defn init-db!"Initialize c3p0 connection pool for semantic search database.Requires MB_PGVECTOR_DB_URL environment variable."[](locking data-source(or @data-source- (let [db-config (build-db-config)- unpooled-ds (jdbc/get-datasource db-config)+ (let [{:keys [jdbc-url pool-props]} (parsed-db-config)+ unpooled-ds (jdbc/get-datasource {:jdbcUrl jdbc-url})pooled-ds (DataSources/pooledDataSource^javax.sql.DataSource unpooled-ds- (connection-pool/map->properties semantic-search-connection-pool-props))]- (log/info "Initializing semantic search connection pool with properties:" semantic-search-connection-pool-props)+ (connection-pool/map->properties pool-props))]+ (log/info "Initializing semantic search connection pool with properties:" pool-props)+ (force shutdown-hook)(reset! data-source pooled-ds)))))(defn test-connection!diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/index.cljindex 6ca731e0025..6bb4dbb45fb 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj@@ -25,6 +25,7 @@[next.jdbc.result-set :as jdbc.rs][toucan2.core :as t2])(:import+ [com.mchange.v2.c3p0 PooledDataSource][java.time Instant LocalDate OffsetDateTime ZonedDateTime][java.util.concurrent ArrayBlockingQueue RejectedExecutionHandler RejectedExecutionException TimeUnit ThreadPoolExecutor][org.postgresql.util PGobject]))@@ -889,6 +890,18 @@[db query](jdbc/plan db (sql-format-quoted query) {:builder-fn jdbc.rs/as-unqualified-lower-maps}))+(defn- warm-connection-pool-async!+ "Warm `db` with one pooled connection on a background thread, unless it already holds an idle one+ (or isn't a c3p0 pool at all, e.g. a plain datasource in tests).+ Fire-and-forget: failures surface on the real query that follows."+ [db]+ (u/ignore-exceptions+ (when (and (instance? PooledDataSource db)+ (zero? (.getNumIdleConnectionsDefaultUser ^PooledDataSource db)))+ (future+ (u/ignore-exceptions+ (with-open [_conn (.getConnection ^PooledDataSource db)]))))))+(defn query-index"Query the index for documents similar to the search string.Returns a map with :results and :raw-count."@@ -899,6 +912,11 @@{:results [] :raw-count 0}(let [timer (u/start-timer)+ ;; Warm the pool concurrently with the embedding round-trip: the pool holds zero idle+ ;; connections by default (see semantic-search.db.datasource), so the first search after an+ ;; idle period would otherwise pay the connection handshake serially on top of the embedding+ ;; latency.+ _ (warm-connection-pool-async! db)embedding (tracing/with-span :search "search.semantic.embedding"{:search.semantic/provider (:provider embedding-model):search.semantic/model-name (:model-name embedding-model)}diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/db_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/db_test.cljindex 703d02155f6..c9ecaa76be6 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/db_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/db_test.clj@@ -48,3 +48,97 @@(semantic.db.datasource/test-connection!)))(finally(reset! semantic.db.datasource/data-source orig-data-source))))))++(def ^:private parse-db-url #'semantic.db.datasource/parse-db-url)++(def ^:private base-url "jdbc:postgresql://localhost:5432/mb_semantic_search")++(deftest parse-db-url-defaults-test+ (testing "a URL with no params leaves the URL untouched and uses the default pool props"+ ;; strict = (not =?) so an unexpected extra/missing prop also fails, not just a wrong value+ (is (= {:jdbc-url base-url+ :pool-props {;; tunable knobs at their defaults+ "maxPoolSize" 5+ "minPoolSize" 0+ "initialPoolSize" 0+ "checkoutTimeout" 10000+ "unreturnedConnectionTimeout" 0+ "debugUnreturnedConnectionStackTraces" false+ "testConnectionOnCheckout" false+ ;; fixed props operators can't override+ "idleConnectionTestPeriod" 60+ "maxIdleTimeExcessConnections" 600+ "maxConnectionAge" 1800+ "acquireIncrement" 1+ "dataSourceName" "metabase-semantic-search-db"}}+ (parse-db-url base-url)))))++(deftest parse-db-url-pool-knob-test+ (testing "a recognized pool knob is pulled off the URL, parsed, and merged over the defaults"+ (is (=? {:jdbc-url base-url+ :pool-props {"maxPoolSize" 12}}+ (parse-db-url (str base-url "?maxPoolSize=12")))))+ (testing "a boolean knob coerces case-insensitively"+ (is (=? {:pool-props {"testConnectionOnCheckout" true}}+ (parse-db-url (str base-url "?testConnectionOnCheckout=TRUE"))))))++(deftest parse-db-url-connection-param-test+ (testing "a recognized Postgres connection param stays on the URL for pgjdbc"+ (is (=? {:jdbc-url (str base-url "?tcpKeepAlive=true")}+ (parse-db-url (str base-url "?tcpKeepAlive=true")))))+ (testing "pool knobs are stripped while connection params + credentials are retained, in order"+ (is (=? {:jdbc-url (str base-url "?user=postgres&tcpKeepAlive=true")+ :pool-props {"maxPoolSize" 8}}+ (parse-db-url (str base-url "?user=postgres&maxPoolSize=8&tcpKeepAlive=true"))))))++(deftest parse-db-url-validation-test+ (testing "an unrecognized param throws rather than being silently ignored by pgjdbc"+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"Unknown pgvector URL parameter maxPoolSizee"+ (parse-db-url (str base-url "?maxPoolSizee=5")))))+ (testing "a malformed pool-knob value throws"+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"Invalid value for pgvector pool parameter maxPoolSize"+ (parse-db-url (str base-url "?maxPoolSize=lots"))))))++(deftest pool-props-applied-by-c3p0-test+ (testing "c3p0 actually understands and applies every pool property name we set"+ ;; We hand c3p0 a Properties map; like pgjdbc, c3p0 silently ignores names it doesn't recognize, so a+ ;; typo in our prop map would quietly fall back to a c3p0 default. Read the effective config back from+ ;; the pool to prove each name took effect. The values below all differ from c3p0's own defaults, so a+ ;; dropped property would surface as a mismatch.+ (when semantic.db.datasource/db-url+ (let [url (str semantic.db.datasource/db-url+ "&minPoolSize=2"+ "&initialPoolSize=2"+ "&maxPoolSize=9"+ "&checkoutTimeout=4321"+ "&unreturnedConnectionTimeout=77"+ "&debugUnreturnedConnectionStackTraces=true"+ "&testConnectionOnCheckout=true")]+ (with-redefs [semantic.db.datasource/db-url url+ semantic.db.datasource/data-source (atom nil)]+ (try+ (semantic.db.datasource/init-db!)+ (let [pool-ds ^PoolBackedDataSource @semantic.db.datasource/data-source+ cpds (.getConnectionPoolDataSource pool-ds)]+ (is (=? {;; tunable knobs supplied on the URL+ :maxPoolSize 9+ :minPoolSize 2+ :initialPoolSize 2+ :checkoutTimeout 4321+ :unreturnedConnectionTimeout 77+ :debugUnreturnedConnectionStackTraces true+ :testConnectionOnCheckout true+ ;; fixed props we always set, on the connection-pool DS+ :idleConnectionTestPeriod 60+ :maxIdleTimeExcessConnections 600+ :maxConnectionAge 1800+ :acquireIncrement 1}+ (bean cpds)))+ ;; dataSourceName is the one fixed prop that lives on the outer pooled DS, not the cpds+ (is (= "metabase-semantic-search-db" (.getDataSourceName pool-ds))))+ (finally+ (semantic.db.datasource/shutdown-db!))))))))