Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Add operator knobs for the pgvector connection pool (#75651)

Chris Truterb934ac131de6
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/db/datasource.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/db/datasource.clj
index b6d7872fc92..b299d7de5bc 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/db/datasource.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/db/datasource.clj
@@ -1,11 +1,16 @@
(ns metabase-enterprise.semantic-search.db.datasource
(:require
+ [clojure.string :as str]
[environ.core :refer [env]]
[metabase.connection-pool :as connection-pool]
+ [metabase.util :as u]
[metabase.util.log :as log]
[next.jdbc :as jdbc])
(:import
- (com.mchange.v2.c3p0 DataSources)))
+ (com.mchange.v2.c3p0 DataSources PooledDataSource)
+ (java.net URLDecoder)
+ (java.nio.charset StandardCharsets)
+ (org.postgresql PGProperty)))
(set! *warn-on-reflection* true)
@@ -17,37 +22,140 @@
"The database URL used to connect to pgvector"
(env :mb-pgvector-db-url))
-;; See metabase.app-db.connection-pool-setup for more details on these properties
-;; TODO: not sure if we need the MetabaseConnectionCustomizer like in the app DB connection pool setup
-(def ^:private semantic-search-connection-pool-props
- "Connection pool properties for semantic search database using c3p0."
+;; All pgvector config — connection *and* connection-pool parameters — rides MB_PGVECTOR_DB_URL, so a
+;; deployment sets everything in one place (the DB secret) with no env-var-vs-URL precedence to reason
+;; about. c3p0 pool params can't ride the JDBC URL into the driver, so we parse them out here (see
+;; parse-db-url) and hand the rest to pgjdbc. pgjdbc *silently ignores* params it doesn't recognize, so we
+;; validate every param against an allowlist and throw on startup rather than let a typo become a
+;; silently-dropped setting. Cf. metabase.app-db.connection-pool-setup for the analogous app-db pool.
+
+;; c3p0 props we always set and don't expose for tuning.
+(def ^:private fixed-pool-props
{"idleConnectionTestPeriod" 60
- "maxIdleTimeExcessConnections" (* 10 60) ; 5 minutes
- "maxConnectionAge" (* 30 60) ; 30 minutes
- "maxPoolSize" 5 ; Small pool to start
- "minPoolSize" 1
- "initialPoolSize" 1
+ "maxIdleTimeExcessConnections" (* 10 60) ; cull excess idle connections after 10 minutes
+ "maxConnectionAge" (* 30 60) ; recycle any connection after 30 minutes
+ ;; c3p0 defaults to 3 per miss; 1 avoids tripling the cold-start burst on the shared RDS.
+ "acquireIncrement" 1
"dataSourceName" "metabase-semantic-search-db"})
-(defn- build-db-config
- "Build database configuration from environment variables"
+(defn- parse-bool [^String s]
+ (case (u/lower-case-en (str/trim s)) "true" true "false" false nil))
+
+;; Overridable c3p0 knobs: name -> [default parse-fn]. Operators override by setting the same key on
+;; MB_PGVECTOR_DB_URL; names match c3p0's own so they map straight onto its docs. parse-fn returns nil on a
+;; malformed value, which we surface as a startup error.
+(def ^:private tunable-pool-props
+ {;; 0 idle connections is what makes a shared RDS viable. Cold start costs one handshake, warmed
+ ;; concurrently with the embedding round-trip (see semantic-search.index/query-index). Set minPoolSize=1
+ ;; to pin a warm connection on a hot instance.
+ "minPoolSize" [0 parse-long]
+ "initialPoolSize" [0 parse-long]
+ "maxPoolSize" [5 parse-long]
+ ;; Fail fast on pool exhaustion (c3p0 default 0 blocks forever); the engine falls back to appdb search
+ ;; on error (see semantic-search.core/results).
+ "checkoutTimeout" [10000 parse-long]
+ ;; c3p0's leak detector, off by default: it destroys connections by checkout *duration*, so any value
+ ;; low enough to catch leaks also kills slow work (reindex DDL, bulk upserts). Enable with the
+ ;; stack-traces knob on a canary to hunt a suspected leak.
+ "unreturnedConnectionTimeout" [0 parse-long]
+ "debugUnreturnedConnectionStackTraces" [false parse-bool]
+ ;; Off by default (idleConnectionTestPeriod covers idle conns); matches app-db. Flip on if the shared
+ ;; RDS drops connections under the pool.
+ "testConnectionOnCheckout" [false parse-bool]})
+
+;; Postgres connection properties pgjdbc recognizes on the classpath; read at runtime so it tracks the
+;; driver version. Anything on the URL that's neither a tunable pool knob nor one of these is a typo.
+;; NB: explicit fn rather than the 1.12 `PGProperty/.getName` method value — eastwood's analyzer can't
+;; parse the latter yet.
+(def ^:private known-connection-params
+ (into #{} (map #(.getName ^PGProperty %)) (PGProperty/values)))
+
+(defn- url-decode ^String [^String s]
+ (URLDecoder/decode s StandardCharsets/UTF_8))
+
+(defn- split-query
+ "Split a JDBC URL into [base pairs], where pairs is a seq of raw [key value] strings.
+ Pairs are left verbatim so connection params pass through to pgjdbc exactly as written; only pool-knob
+ values are decoded, in parse-db-url."
+ [^String url]
+ (let [[base query] (str/split url #"\?" 2)]
+ ;; split each pair on its first '=' only, so a value may itself contain '=' (e.g. options=-c foo=bar)
+ [base (for [pair (some-> query (str/split #"&"))
+ :when (seq pair)
+ :let [[k v] (str/split pair #"=" 2)]]
+ [k (or v "")])]))
+
+(defn- parse-db-url
+ "Parse a pgvector JDBC URL into {:jdbc-url ... :pool-props ...}, or throw if it carries an unrecognized
+ parameter."
+ [^String url]
+ (let [[base pairs] (split-query url)
+ default-pool (merge fixed-pool-props
+ (update-vals tunable-pool-props first))
+ {:keys [pool conn]}
+ (reduce
+ (fn [acc [k raw-v]]
+ (cond
+ ;; a c3p0 knob: coerce and apply to the pool (pgjdbc can't read these off the URL)
+ (contains? tunable-pool-props k)
+ (let [parse (second (tunable-pool-props k))
+ parsed (parse (url-decode raw-v))]
+ (when (nil? parsed)
+ (throw (ex-info (format "Invalid value for pgvector pool parameter %s: %s" k raw-v)
+ {:param k, :value raw-v})))
+ (assoc-in acc [:pool k] parsed))
+
+ ;; a recognized pgjdbc connection property: leave it on the URL for the driver
+ (contains? known-connection-params k)
+ (update acc :conn conj (str k "=" raw-v))
+
+ ;; pgjdbc would silently ignore anything else, so treat it as a typo and fail loudly
+ :else
+ (throw (ex-info (format (str "Unknown pgvector URL parameter %s. Expected a c3p0 pool knob "
+ "(%s) or a Postgres connection property.")
+ k (str/join ", " (sort (keys tunable-pool-props))))
+ {:param k}))))
+ {:pool default-pool, :conn []}
+ pairs)]
+ {:jdbc-url (cond-> base (seq conn) (str "?" (str/join "&" conn)))
+ :pool-props pool}))
+
+(defn- parsed-db-config
+ "Parse [[db-url]] into {:jdbc-url ... :pool-props ...}, or throw if it is unset."
[]
(if db-url
- {:jdbcUrl db-url}
+ (parse-db-url db-url)
(throw (ex-info "MB_PGVECTOR_DB_URL environment variable is required for semantic search" {}))))
+(defn shutdown-db!
+ "Close the pooled data source (if any) and clear it, releasing all server-side connections.
+ Safe to call when the pool was never initialized; a later [[init-db!]] builds a fresh pool."
+ []
+ (locking data-source
+ (when-let [ds @data-source]
+ (reset! data-source nil)
+ (log/info "Closing semantic search connection pool")
+ (.close ^PooledDataSource ds))))
+
+;; Tests redef [[data-source]] and close their pools themselves; this hook only sees the root binding,
+;; so it closes at most the production pool.
+(defonce ^:private shutdown-hook
+ (delay (.addShutdownHook (Runtime/getRuntime)
+ (Thread. ^Runnable shutdown-db! "semantic-search-pool-shutdown"))))
+
(defn init-db!
"Initialize c3p0 connection pool for semantic search database.
Requires MB_PGVECTOR_DB_URL environment variable."
[]
(locking data-source
(or @data-source
- (let [db-config (build-db-config)
- unpooled-ds (jdbc/get-datasource db-config)
+ (let [{:keys [jdbc-url pool-props]} (parsed-db-config)
+ unpooled-ds (jdbc/get-datasource {:jdbcUrl jdbc-url})
pooled-ds (DataSources/pooledDataSource
^javax.sql.DataSource unpooled-ds
- (connection-pool/map->properties semantic-search-connection-pool-props))]
- (log/info "Initializing semantic search connection pool with properties:" semantic-search-connection-pool-props)
+ (connection-pool/map->properties pool-props))]
+ (log/info "Initializing semantic search connection pool with properties:" pool-props)
+ (force shutdown-hook)
(reset! data-source pooled-ds)))))
(defn test-connection!
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj
index 6ca731e0025..6bb4dbb45fb 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj
@@ -25,6 +25,7 @@
[next.jdbc.result-set :as jdbc.rs]
[toucan2.core :as t2])
(:import
+ [com.mchange.v2.c3p0 PooledDataSource]
[java.time Instant LocalDate OffsetDateTime ZonedDateTime]
[java.util.concurrent ArrayBlockingQueue RejectedExecutionHandler RejectedExecutionException TimeUnit ThreadPoolExecutor]
[org.postgresql.util PGobject]))
@@ -889,6 +890,18 @@
[db query]
(jdbc/plan db (sql-format-quoted query) {:builder-fn jdbc.rs/as-unqualified-lower-maps}))
+(defn- warm-connection-pool-async!
+ "Warm `db` with one pooled connection on a background thread, unless it already holds an idle one
+ (or isn't a c3p0 pool at all, e.g. a plain datasource in tests).
+ Fire-and-forget: failures surface on the real query that follows."
+ [db]
+ (u/ignore-exceptions
+ (when (and (instance? PooledDataSource db)
+ (zero? (.getNumIdleConnectionsDefaultUser ^PooledDataSource db)))
+ (future
+ (u/ignore-exceptions
+ (with-open [_conn (.getConnection ^PooledDataSource db)]))))))
+
(defn query-index
"Query the index for documents similar to the search string.
Returns a map with :results and :raw-count."
@@ -899,6 +912,11 @@
{:results [] :raw-count 0}
(let [timer (u/start-timer)
+ ;; Warm the pool concurrently with the embedding round-trip: the pool holds zero idle
+ ;; connections by default (see semantic-search.db.datasource), so the first search after an
+ ;; idle period would otherwise pay the connection handshake serially on top of the embedding
+ ;; latency.
+ _ (warm-connection-pool-async! db)
embedding (tracing/with-span :search "search.semantic.embedding"
{:search.semantic/provider (:provider embedding-model)
:search.semantic/model-name (:model-name embedding-model)}
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/db_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/db_test.clj
index 703d02155f6..c9ecaa76be6 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/db_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/db_test.clj
@@ -48,3 +48,97 @@
(semantic.db.datasource/test-connection!)))
(finally
(reset! semantic.db.datasource/data-source orig-data-source))))))
+
+(def ^:private parse-db-url #'semantic.db.datasource/parse-db-url)
+
+(def ^:private base-url "jdbc:postgresql://localhost:5432/mb_semantic_search")
+
+(deftest parse-db-url-defaults-test
+ (testing "a URL with no params leaves the URL untouched and uses the default pool props"
+ ;; strict = (not =?) so an unexpected extra/missing prop also fails, not just a wrong value
+ (is (= {:jdbc-url base-url
+ :pool-props {;; tunable knobs at their defaults
+ "maxPoolSize" 5
+ "minPoolSize" 0
+ "initialPoolSize" 0
+ "checkoutTimeout" 10000
+ "unreturnedConnectionTimeout" 0
+ "debugUnreturnedConnectionStackTraces" false
+ "testConnectionOnCheckout" false
+ ;; fixed props operators can't override
+ "idleConnectionTestPeriod" 60
+ "maxIdleTimeExcessConnections" 600
+ "maxConnectionAge" 1800
+ "acquireIncrement" 1
+ "dataSourceName" "metabase-semantic-search-db"}}
+ (parse-db-url base-url)))))
+
+(deftest parse-db-url-pool-knob-test
+ (testing "a recognized pool knob is pulled off the URL, parsed, and merged over the defaults"
+ (is (=? {:jdbc-url base-url
+ :pool-props {"maxPoolSize" 12}}
+ (parse-db-url (str base-url "?maxPoolSize=12")))))
+ (testing "a boolean knob coerces case-insensitively"
+ (is (=? {:pool-props {"testConnectionOnCheckout" true}}
+ (parse-db-url (str base-url "?testConnectionOnCheckout=TRUE"))))))
+
+(deftest parse-db-url-connection-param-test
+ (testing "a recognized Postgres connection param stays on the URL for pgjdbc"
+ (is (=? {:jdbc-url (str base-url "?tcpKeepAlive=true")}
+ (parse-db-url (str base-url "?tcpKeepAlive=true")))))
+ (testing "pool knobs are stripped while connection params + credentials are retained, in order"
+ (is (=? {:jdbc-url (str base-url "?user=postgres&tcpKeepAlive=true")
+ :pool-props {"maxPoolSize" 8}}
+ (parse-db-url (str base-url "?user=postgres&maxPoolSize=8&tcpKeepAlive=true"))))))
+
+(deftest parse-db-url-validation-test
+ (testing "an unrecognized param throws rather than being silently ignored by pgjdbc"
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"Unknown pgvector URL parameter maxPoolSizee"
+ (parse-db-url (str base-url "?maxPoolSizee=5")))))
+ (testing "a malformed pool-knob value throws"
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"Invalid value for pgvector pool parameter maxPoolSize"
+ (parse-db-url (str base-url "?maxPoolSize=lots"))))))
+
+(deftest pool-props-applied-by-c3p0-test
+ (testing "c3p0 actually understands and applies every pool property name we set"
+ ;; We hand c3p0 a Properties map; like pgjdbc, c3p0 silently ignores names it doesn't recognize, so a
+ ;; typo in our prop map would quietly fall back to a c3p0 default. Read the effective config back from
+ ;; the pool to prove each name took effect. The values below all differ from c3p0's own defaults, so a
+ ;; dropped property would surface as a mismatch.
+ (when semantic.db.datasource/db-url
+ (let [url (str semantic.db.datasource/db-url
+ "&minPoolSize=2"
+ "&initialPoolSize=2"
+ "&maxPoolSize=9"
+ "&checkoutTimeout=4321"
+ "&unreturnedConnectionTimeout=77"
+ "&debugUnreturnedConnectionStackTraces=true"
+ "&testConnectionOnCheckout=true")]
+ (with-redefs [semantic.db.datasource/db-url url
+ semantic.db.datasource/data-source (atom nil)]
+ (try
+ (semantic.db.datasource/init-db!)
+ (let [pool-ds ^PoolBackedDataSource @semantic.db.datasource/data-source
+ cpds (.getConnectionPoolDataSource pool-ds)]
+ (is (=? {;; tunable knobs supplied on the URL
+ :maxPoolSize 9
+ :minPoolSize 2
+ :initialPoolSize 2
+ :checkoutTimeout 4321
+ :unreturnedConnectionTimeout 77
+ :debugUnreturnedConnectionStackTraces true
+ :testConnectionOnCheckout true
+ ;; fixed props we always set, on the connection-pool DS
+ :idleConnectionTestPeriod 60
+ :maxIdleTimeExcessConnections 600
+ :maxConnectionAge 1800
+ :acquireIncrement 1}
+ (bean cpds)))
+ ;; dataSourceName is the one fixed prop that lives on the outer pooled DS, not the cpds
+ (is (= "metabase-semantic-search-db" (.getDataSourceName pool-ds))))
+ (finally
+ (semantic.db.datasource/shutdown-db!))))))))