Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Prime app-db connection-pool before use (#81601)

metambend0cbe49c68de
diff --git a/src/metabase/app_db/connection_pool_setup.clj b/src/metabase/app_db/connection_pool_setup.clj
index 4a08cc5821e..586ff129755 100644
--- a/src/metabase/app_db/connection_pool_setup.clj
+++ b/src/metabase/app_db/connection_pool_setup.clj
@@ -4,11 +4,12 @@
[java-time.api :as t]
[metabase.config.core :as config]
[metabase.connection-pool :as connection-pool]
+ [metabase.util.log :as log]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
[potemkin :as p])
(:import
- (com.mchange.v2.c3p0 ConnectionCustomizer PoolBackedDataSource)))
+ (com.mchange.v2.c3p0 ConnectionCustomizer DataSources PoolBackedDataSource)))
(set! *warn-on-reflection* true)
@@ -182,6 +183,22 @@
;; the first place."
3600)})
+;; Wall-clock bound on how long we wait for a newly-built pool to hand out its first connection when priming it (see
+;; `connection-pool-data-source`). Chosen so a slow first connect (cold DNS, TLS handshake, cross-region
+;; latency) does not trip us into a false-positive rebuild.
+(def ^:private prime-timeout-ms 3000)
+
+(defn- prime-pool!
+ "Do a bounded-wall-clock `.getConnection()` on `pool` to detect the born-wedged c3p0 state described in
+ https://github.com/metabase/metabase/issues/81440 (HelperThreads idle in `Object.wait()`, no acquire task ever
+ dispatched). Returns true if the pool handed out a connection within `timeout-ms`, false otherwise."
+ [^javax.sql.DataSource pool ^long timeout-ms]
+ (let [fut (future (with-open [_ (.getConnection pool)] true))]
+ (try
+ (true? (deref fut timeout-ms false))
+ (catch Throwable _ false)
+ (finally (future-cancel fut)))))
+
(mu/defn connection-pool-data-source :- (ms/InstanceOfClass PoolBackedDataSource)
"Create a connection pool [[javax.sql.DataSource]] from an unpooled [[javax.sql.DataSource]] `data-source`. If
`data-source` is already pooled, this will return `data-source` as-is, ignoring `props-overrides`.
@@ -196,10 +213,24 @@
(let [ds-name (format "metabase-%s-app-db" (name db-type))
pool-props (merge (application-db-connection-pool-props)
{"dataSourceName" ds-name}
- props-overrides)]
- (com.mchange.v2.c3p0.DataSources/pooledDataSource
- data-source
- (connection-pool/map->properties pool-props))))))
+ props-overrides)
+ build (fn [] (DataSources/pooledDataSource
+ data-source
+ (connection-pool/map->properties pool-props)))
+ pool (build)]
+ (if (prime-pool! pool prime-timeout-ms)
+ pool
+ ;; The pool did not respond to priming -- probably the c3p0 startup race in #81440. Destroy and try once
+ ;; more; if that also times out, log and hand the (possibly wedged) pool back so startup proceeds.
+ (do
+ (log/warnf "%s pool did not hand out a connection within %d ms; rebuilding (see #81440)"
+ (get pool-props "dataSourceName") prime-timeout-ms)
+ (DataSources/destroy pool)
+ (let [pool' (build)]
+ (when-not (prime-pool! pool' prime-timeout-ms)
+ (log/warnf "%s pool still unresponsive after rebuild; startup continuing (see #81440)"
+ (get pool-props "dataSourceName")))
+ pool')))))))
(def ^:private default-quartz-max-pool-size 5)
diff --git a/test/metabase/app_db/connection_pool_setup_test.clj b/test/metabase/app_db/connection_pool_setup_test.clj
index 52036814921..83ad4f1aaa9 100644
--- a/test/metabase/app_db/connection_pool_setup_test.clj
+++ b/test/metabase/app_db/connection_pool_setup_test.clj
@@ -179,6 +179,29 @@
(is (= 99 (get (#'mdb.connection-pool-setup/application-db-connection-pool-props)
"unreturnedConnectionTimeout"))))))
+(defn- mock-data-source
+ "A `javax.sql.DataSource` whose `.getConnection` calls `f` and expects it to return a `java.sql.Connection`
+ (typically a no-op `reify` in these tests)."
+ ^javax.sql.DataSource [f]
+ (reify javax.sql.DataSource
+ (getConnection [_] (f))))
+
+(deftest prime-pool!-test
+ (testing "returns true when the data source hands out a connection within the timeout"
+ (let [ds (mock-data-source #(reify java.sql.Connection (close [_])))]
+ (is (true? (#'mdb.connection-pool-setup/prime-pool! ds 5000)))))
+ (testing "returns false when getConnection blocks past the timeout, and does not wait for it"
+ ;; Regression guard for #81440: without a wall-clock bound on the priming acquire, a wedged pool would hang
+ ;; startup for the full checkoutTimeout (30s by default).
+ (let [ds (mock-data-source (fn []
+ (Thread/sleep 60000)
+ (throw (RuntimeException. "should have been interrupted"))))
+ timer (u/start-timer)
+ result (#'mdb.connection-pool-setup/prime-pool! ds 100)]
+ (is (false? result))
+ (is (< (u/since-ms timer) 1000)
+ "prime-pool! must return within a small multiple of the timeout, not wait for getConnection"))))
+
(deftest reset-read-only-test
(testing "For Postgres app DBs, we should be executing `DISCARD ALL` when checking in a connection to reset state including read-only"
(when (= (app-db/db-type) :postgres)