Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Close empty pool after failed connection (#84565)

Riley Thompson8ccfb3acc5ae
diff --git a/src/metabase/driver/sql_jdbc/connection.clj b/src/metabase/driver/sql_jdbc/connection.clj
index 9fca130a22e..f06ac35fac0 100644
--- a/src/metabase/driver/sql_jdbc/connection.clj
+++ b/src/metabase/driver/sql_jdbc/connection.clj
@@ -22,8 +22,8 @@
[metabase.warehouses.schema :as warehouses.schema]
[potemkin :as p])
(:import
- (com.mchange.v2.c3p0 C3P0ProxyConnection DataSources)
- (java.sql Connection)
+ (com.mchange.v2.c3p0 C3P0ProxyConnection DataSources PooledDataSource)
+ (java.sql Connection SQLException)
(javax.sql DataSource)
(org.apache.logging.log4j Level)))
@@ -373,6 +373,32 @@
(destroy-pool! db-id pool-spec)
(swap! pool-cache-key->jdbc-spec-hash dissoc cache-key))))
+(defn- empty-pool?
+ [data-source]
+ (and (instance? PooledDataSource data-source)
+ (try
+ (zero? (.getNumConnectionsDefaultUser ^PooledDataSource data-source))
+ (catch SQLException _
+ false))))
+
+(defn invalidate-empty-pool!
+ "Closes the cached pool for a given `data-source` if it has no connections. This should be called
+ after a checkout from a pool fails, otherwise c3p0 will make repeated requests which can lock
+ out certain DBs (#84206). We don't use [[pool-cache-key]] here to avoid looking up the database
+ object from the app DB, which could fail or have a race condition across multiple pods."
+ [data-source]
+ (when (empty-pool? data-source)
+ (doseq [[cache-key pool-spec] @pool-cache-key->connection-pool
+ :when (identical? (:datasource pool-spec) data-source)
+ :let [db-id (first cache-key)]]
+ (log/warnf "Could not acquire a connection for database %d; closing its empty connection pool" db-id)
+ ;; do not remove a newer pool that another thread put in the cache
+ (swap! pool-cache-key->connection-pool
+ #(cond-> %
+ (identical? (get % cache-key) pool-spec) (dissoc cache-key)))
+ (destroy-pool! db-id pool-spec)
+ (swap! pool-cache-key->jdbc-spec-hash dissoc cache-key))))
+
(defn- log-ssh-tunnel-reconnect-msg! [db-id]
(log/warn (u/format-color :red "ssh tunnel for database %s looks closed; marking pool invalid to reopen it" db-id))
nil)
diff --git a/src/metabase/driver/sql_jdbc/execute.clj b/src/metabase/driver/sql_jdbc/execute.clj
index 80099ebe548..bfae73bf45f 100644
--- a/src/metabase/driver/sql_jdbc/execute.clj
+++ b/src/metabase/driver/sql_jdbc/execute.clj
@@ -378,6 +378,11 @@
(.getConnection conn-data-source)
(catch Throwable e
(let [timed-out? (connection-checkout-timeout? e)]
+ (when-not timed-out?
+ (try
+ (sql-jdbc.conn/invalidate-empty-pool! conn-data-source)
+ (catch Exception e2
+ (log/warnf e2 "Error closing the connection pool for database %s" (u/id db-or-id-or-spec)))))
(throw (ex-info (if timed-out?
(tru "Too many queries are running. Please try again in a moment.")
(tru "Unable to connect to the database: {0}" (ex-message e)))
diff --git a/test/metabase/driver/sql_jdbc/execute_test.clj b/test/metabase/driver/sql_jdbc/execute_test.clj
index 1815ea7f4b9..8c28526ab90 100644
--- a/test/metabase/driver/sql_jdbc/execute_test.clj
+++ b/test/metabase/driver/sql_jdbc/execute_test.clj
@@ -6,6 +6,7 @@
[metabase.driver :as driver]
[metabase.driver.connection :as driver.conn]
[metabase.driver.h2 :as h2]
+ [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
[metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
@@ -275,6 +276,35 @@
(is (true? (full? (reify com.mchange.v2.c3p0.PooledDataSource
(getNumThreadsAwaitingCheckoutDefaultUser [_] 3)))))))))
+(deftest failed-checkout-destroys-empty-pool-test
+ (testing "When a checkout fails, do-with-resolved-connection destroys the pool only if the pool has no Connections"
+ (doseq [[num-connections destroyed?] [[0 true] [1 false]]]
+ (let [db-id Integer/MAX_VALUE
+ cache-key [db-id :default]
+ data-source (reify com.mchange.v2.c3p0.PooledDataSource
+ (getConnection [_]
+ (throw (java.sql.SQLException. "password authentication failed")))
+ (getNumConnectionsDefaultUser [_] num-connections))
+ pool-spec {:datasource data-source}
+ destroyed (atom [])
+ pool-cache @#'sql-jdbc.conn/pool-cache-key->connection-pool]
+ (swap! pool-cache assoc cache-key pool-spec)
+ (try
+ (mt/with-dynamic-fn-redefs [sql-jdbc.execute/do-with-resolved-connection-data-source
+ (fn [_driver _db-or-id-or-spec _options] data-source)
+ sql-jdbc.conn/destroy-pool!
+ (fn [id spec] (swap! destroyed conj [id spec]))]
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"Unable to connect to the database"
+ (sql-jdbc.execute/do-with-resolved-connection :h2 db-id {} (fn [_conn] nil))))
+ (is (= (if destroyed? [[db-id pool-spec]] [])
+ @destroyed))
+ (is (= (not destroyed?)
+ (contains? @pool-cache cache-key))))
+ (finally
+ (swap! pool-cache dissoc cache-key)))))))
+
(deftest connection-pool-full-checkout-queue-returns-503-test
(testing "When the checkout queue is full, additional queries fail fast with a retriable HTTP 503"
(mt/test-drivers (mt/normal-driver-select {:+parent :sql-jdbc})