The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Close empty pool after failed connection (#84565)
Riley Thompson8ccfb3acc5ae
diff --git a/src/metabase/driver/sql_jdbc/connection.clj b/src/metabase/driver/sql_jdbc/connection.cljindex 9fca130a22e..f06ac35fac0 100644--- a/src/metabase/driver/sql_jdbc/connection.clj+++ b/src/metabase/driver/sql_jdbc/connection.clj@@ -22,8 +22,8 @@[metabase.warehouses.schema :as warehouses.schema][potemkin :as p])(:import- (com.mchange.v2.c3p0 C3P0ProxyConnection DataSources)- (java.sql Connection)+ (com.mchange.v2.c3p0 C3P0ProxyConnection DataSources PooledDataSource)+ (java.sql Connection SQLException)(javax.sql DataSource)(org.apache.logging.log4j Level)))@@ -373,6 +373,32 @@(destroy-pool! db-id pool-spec)(swap! pool-cache-key->jdbc-spec-hash dissoc cache-key))))+(defn- empty-pool?+ [data-source]+ (and (instance? PooledDataSource data-source)+ (try+ (zero? (.getNumConnectionsDefaultUser ^PooledDataSource data-source))+ (catch SQLException _+ false))))++(defn invalidate-empty-pool!+ "Closes the cached pool for a given `data-source` if it has no connections. This should be called+ after a checkout from a pool fails, otherwise c3p0 will make repeated requests which can lock+ out certain DBs (#84206). We don't use [[pool-cache-key]] here to avoid looking up the database+ object from the app DB, which could fail or have a race condition across multiple pods."+ [data-source]+ (when (empty-pool? data-source)+ (doseq [[cache-key pool-spec] @pool-cache-key->connection-pool+ :when (identical? (:datasource pool-spec) data-source)+ :let [db-id (first cache-key)]]+ (log/warnf "Could not acquire a connection for database %d; closing its empty connection pool" db-id)+ ;; do not remove a newer pool that another thread put in the cache+ (swap! pool-cache-key->connection-pool+ #(cond-> %+ (identical? (get % cache-key) pool-spec) (dissoc cache-key)))+ (destroy-pool! db-id pool-spec)+ (swap! pool-cache-key->jdbc-spec-hash dissoc cache-key))))+(defn- log-ssh-tunnel-reconnect-msg! [db-id](log/warn (u/format-color :red "ssh tunnel for database %s looks closed; marking pool invalid to reopen it" db-id))nil)diff --git a/src/metabase/driver/sql_jdbc/execute.clj b/src/metabase/driver/sql_jdbc/execute.cljindex 80099ebe548..bfae73bf45f 100644--- a/src/metabase/driver/sql_jdbc/execute.clj+++ b/src/metabase/driver/sql_jdbc/execute.clj@@ -378,6 +378,11 @@(.getConnection conn-data-source)(catch Throwable e(let [timed-out? (connection-checkout-timeout? e)]+ (when-not timed-out?+ (try+ (sql-jdbc.conn/invalidate-empty-pool! conn-data-source)+ (catch Exception e2+ (log/warnf e2 "Error closing the connection pool for database %s" (u/id db-or-id-or-spec)))))(throw (ex-info (if timed-out?(tru "Too many queries are running. Please try again in a moment.")(tru "Unable to connect to the database: {0}" (ex-message e)))diff --git a/test/metabase/driver/sql_jdbc/execute_test.clj b/test/metabase/driver/sql_jdbc/execute_test.cljindex 1815ea7f4b9..8c28526ab90 100644--- a/test/metabase/driver/sql_jdbc/execute_test.clj+++ b/test/metabase/driver/sql_jdbc/execute_test.clj@@ -6,6 +6,7 @@[metabase.driver :as driver][metabase.driver.connection :as driver.conn][metabase.driver.h2 :as h2]+ [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn][metabase.driver.sql-jdbc.execute :as sql-jdbc.execute][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]@@ -275,6 +276,35 @@(is (true? (full? (reify com.mchange.v2.c3p0.PooledDataSource(getNumThreadsAwaitingCheckoutDefaultUser [_] 3)))))))))+(deftest failed-checkout-destroys-empty-pool-test+ (testing "When a checkout fails, do-with-resolved-connection destroys the pool only if the pool has no Connections"+ (doseq [[num-connections destroyed?] [[0 true] [1 false]]]+ (let [db-id Integer/MAX_VALUE+ cache-key [db-id :default]+ data-source (reify com.mchange.v2.c3p0.PooledDataSource+ (getConnection [_]+ (throw (java.sql.SQLException. "password authentication failed")))+ (getNumConnectionsDefaultUser [_] num-connections))+ pool-spec {:datasource data-source}+ destroyed (atom [])+ pool-cache @#'sql-jdbc.conn/pool-cache-key->connection-pool]+ (swap! pool-cache assoc cache-key pool-spec)+ (try+ (mt/with-dynamic-fn-redefs [sql-jdbc.execute/do-with-resolved-connection-data-source+ (fn [_driver _db-or-id-or-spec _options] data-source)+ sql-jdbc.conn/destroy-pool!+ (fn [id spec] (swap! destroyed conj [id spec]))]+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"Unable to connect to the database"+ (sql-jdbc.execute/do-with-resolved-connection :h2 db-id {} (fn [_conn] nil))))+ (is (= (if destroyed? [[db-id pool-spec]] [])+ @destroyed))+ (is (= (not destroyed?)+ (contains? @pool-cache cache-key))))+ (finally+ (swap! pool-cache dissoc cache-key)))))))+(deftest connection-pool-full-checkout-queue-returns-503-test(testing "When the checkout queue is full, additional queries fail fast with a retriable HTTP 503"(mt/test-drivers (mt/normal-driver-select {:+parent :sql-jdbc})