The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Prime app-db connection-pool before use (#81601)
metambend0cbe49c68de
diff --git a/src/metabase/app_db/connection_pool_setup.clj b/src/metabase/app_db/connection_pool_setup.cljindex 4a08cc5821e..586ff129755 100644--- a/src/metabase/app_db/connection_pool_setup.clj+++ b/src/metabase/app_db/connection_pool_setup.clj@@ -4,11 +4,12 @@[java-time.api :as t][metabase.config.core :as config][metabase.connection-pool :as connection-pool]+ [metabase.util.log :as log][metabase.util.malli :as mu][metabase.util.malli.schema :as ms][potemkin :as p])(:import- (com.mchange.v2.c3p0 ConnectionCustomizer PoolBackedDataSource)))+ (com.mchange.v2.c3p0 ConnectionCustomizer DataSources PoolBackedDataSource)))(set! *warn-on-reflection* true)@@ -182,6 +183,22 @@;; the first place."3600)})+;; Wall-clock bound on how long we wait for a newly-built pool to hand out its first connection when priming it (see+;; `connection-pool-data-source`). Chosen so a slow first connect (cold DNS, TLS handshake, cross-region+;; latency) does not trip us into a false-positive rebuild.+(def ^:private prime-timeout-ms 3000)++(defn- prime-pool!+ "Do a bounded-wall-clock `.getConnection()` on `pool` to detect the born-wedged c3p0 state described in+ https://github.com/metabase/metabase/issues/81440 (HelperThreads idle in `Object.wait()`, no acquire task ever+ dispatched). Returns true if the pool handed out a connection within `timeout-ms`, false otherwise."+ [^javax.sql.DataSource pool ^long timeout-ms]+ (let [fut (future (with-open [_ (.getConnection pool)] true))]+ (try+ (true? (deref fut timeout-ms false))+ (catch Throwable _ false)+ (finally (future-cancel fut)))))+(mu/defn connection-pool-data-source :- (ms/InstanceOfClass PoolBackedDataSource)"Create a connection pool [[javax.sql.DataSource]] from an unpooled [[javax.sql.DataSource]] `data-source`. If`data-source` is already pooled, this will return `data-source` as-is, ignoring `props-overrides`.@@ -196,10 +213,24 @@(let [ds-name (format "metabase-%s-app-db" (name db-type))pool-props (merge (application-db-connection-pool-props){"dataSourceName" ds-name}- props-overrides)]- (com.mchange.v2.c3p0.DataSources/pooledDataSource- data-source- (connection-pool/map->properties pool-props))))))+ props-overrides)+ build (fn [] (DataSources/pooledDataSource+ data-source+ (connection-pool/map->properties pool-props)))+ pool (build)]+ (if (prime-pool! pool prime-timeout-ms)+ pool+ ;; The pool did not respond to priming -- probably the c3p0 startup race in #81440. Destroy and try once+ ;; more; if that also times out, log and hand the (possibly wedged) pool back so startup proceeds.+ (do+ (log/warnf "%s pool did not hand out a connection within %d ms; rebuilding (see #81440)"+ (get pool-props "dataSourceName") prime-timeout-ms)+ (DataSources/destroy pool)+ (let [pool' (build)]+ (when-not (prime-pool! pool' prime-timeout-ms)+ (log/warnf "%s pool still unresponsive after rebuild; startup continuing (see #81440)"+ (get pool-props "dataSourceName")))+ pool')))))))(def ^:private default-quartz-max-pool-size 5)diff --git a/test/metabase/app_db/connection_pool_setup_test.clj b/test/metabase/app_db/connection_pool_setup_test.cljindex 52036814921..83ad4f1aaa9 100644--- a/test/metabase/app_db/connection_pool_setup_test.clj+++ b/test/metabase/app_db/connection_pool_setup_test.clj@@ -179,6 +179,29 @@(is (= 99 (get (#'mdb.connection-pool-setup/application-db-connection-pool-props)"unreturnedConnectionTimeout"))))))+(defn- mock-data-source+ "A `javax.sql.DataSource` whose `.getConnection` calls `f` and expects it to return a `java.sql.Connection`+ (typically a no-op `reify` in these tests)."+ ^javax.sql.DataSource [f]+ (reify javax.sql.DataSource+ (getConnection [_] (f))))++(deftest prime-pool!-test+ (testing "returns true when the data source hands out a connection within the timeout"+ (let [ds (mock-data-source #(reify java.sql.Connection (close [_])))]+ (is (true? (#'mdb.connection-pool-setup/prime-pool! ds 5000)))))+ (testing "returns false when getConnection blocks past the timeout, and does not wait for it"+ ;; Regression guard for #81440: without a wall-clock bound on the priming acquire, a wedged pool would hang+ ;; startup for the full checkoutTimeout (30s by default).+ (let [ds (mock-data-source (fn []+ (Thread/sleep 60000)+ (throw (RuntimeException. "should have been interrupted"))))+ timer (u/start-timer)+ result (#'mdb.connection-pool-setup/prime-pool! ds 100)]+ (is (false? result))+ (is (< (u/since-ms timer) 1000)+ "prime-pool! must return within a small multiple of the timeout, not wait for getConnection"))))+(deftest reset-read-only-test(testing "For Postgres app DBs, we should be executing `DISCARD ALL` when checking in a connection to reset state including read-only"(when (= (app-db/db-type) :postgres)