Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Fix "The result set is closed" flaky reuse of connections (#80033)

Ben Grabowa8db179087eb
diff --git a/docs/developers-guide/driver-changelog.md b/docs/developers-guide/driver-changelog.md
index 6b0a37a3766..c1f306e1ea3 100644
--- a/docs/developers-guide/driver-changelog.md
+++ b/docs/developers-guide/driver-changelog.md
@@ -6,6 +6,12 @@ title: Driver interface changelog
## Metabase 0.64.0
+- `metabase.driver.sql-jdbc.execute/cancelation-poisons-connection?` `[driver]` -- whether canceling a `Statement`
+ leaves the `Connection` unfit for the next query, so that it must be discarded rather than returned to the
+ connection pool. The query processor cancels a `Statement` whenever reduction stops before the `ResultSet` runs out
+ of rows, which abandons a result set the server is still producing. Defaults to `false`, because most drivers
+ recover on their own; `:sqlserver` overrides it to `true`.
+
- `metabase.driver/validate-impersonated-query` `[driver query]` now has a `:sql-jdbc` implementation that
enforces, for every JDBC driver, that a connection-impersonated native query is a single statement.
diff --git a/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj b/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj
index 44adb046591..b860ab085a4 100644
--- a/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj
+++ b/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj
@@ -1047,6 +1047,13 @@
(.close stmt)
(throw e)))))
+(defmethod sql-jdbc.execute/cancelation-poisons-connection? :sqlserver
+ [_driver]
+ ;; `.cancel` sends an out-of-band TDS attention packet. Its acknowledgement is not drained before the Connection is
+ ;; checked back into the pool, and it surfaces later as `The result set is closed.` while an unrelated query is
+ ;; reading rows on the recycled Connection.
+ true)
+
(defmethod sql.qp/inline-value [:sqlserver LocalDate]
[_ ^LocalDate t]
;; datefromparts(year, month, day)
diff --git a/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj b/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj
index 2bb59e61db4..799a51d04d6 100644
--- a/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj
+++ b/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj
@@ -1229,3 +1229,18 @@
hosts #(set (driver/connection-parameter-hosts :sqlserver %))]
(is (contains? (hosts (assoc details :additional-options "serverName=10.0.0.1")) "10.0.0.1"))
(is (not (contains? (hosts details) "10.0.0.1"))))))
+
+(deftest cancelation-poisons-connection-test
+ (testing "discarding the Connection a query canceled on leaves the pool able to serve later queries (#39018)"
+ (mt/test-driver :sqlserver
+ (is (true? (sql-jdbc.execute/cancelation-poisons-connection? :sqlserver))
+ "SQL Server does not recover from a cancelation on its own, so the Connection must not be recycled")
+ (letfn [(rows [table n]
+ (let [mp (mt/metadata-provider)]
+ (cond-> (lib/query mp (lib.metadata/table mp (mt/id table)))
+ n (lib/limit n)
+ true (-> qp/process-query mt/rows))))]
+ ;; stopping at the limit leaves the statement producing, which is what triggers the cancel-and-discard
+ (is (= 4 (count (rows :venues 4))))
+ (testing "and a later query reading every row still succeeds"
+ (is (= 1000 (count (rows :checkins nil)))))))))
diff --git a/src/metabase/driver/sql_jdbc/connection.clj b/src/metabase/driver/sql_jdbc/connection.clj
index 1e745ff609b..f41c9a18962 100644
--- a/src/metabase/driver/sql_jdbc/connection.clj
+++ b/src/metabase/driver/sql_jdbc/connection.clj
@@ -21,7 +21,8 @@
[metabase.util.performance :refer [get-in mapv select-keys]]
[potemkin :as p])
(:import
- (com.mchange.v2.c3p0 DataSources)
+ (com.mchange.v2.c3p0 C3P0ProxyConnection DataSources)
+ (java.sql Connection)
(javax.sql DataSource)
(org.apache.logging.log4j Level)))
@@ -544,3 +545,36 @@
(defmethod driver/connection-spec :sql-jdbc [_driver db]
(db->pooled-connection-spec db))
+
+(def ^:private raw-connection-close-method
+ ;; c3p0 exposes the pooled physical Connection only through `rawConnectionOperation`, which names the operation as a
+ ;; `java.lang.reflect.Method`. `unwrap` is not an alternative: c3p0 delegates it to the driver, and Hive's throws.
+ (.getMethod Connection "close" (make-array Class 0)))
+
+(defn discard-pooled-connection!
+ "Destroy the physical Connection behind a c3p0 proxy, so the pool acquires a fresh one rather than handing this one
+ to the next query. Use for a Connection left in a state the pool cannot reset, such as after canceling a Statement
+ on a driver where that leaves unread protocol state on the wire.
+
+ Only the caller that owns `conn` may call this: the pool has no way to tell the difference between a Connection
+ whose borrower is finished with it and one that is still in use.
+
+ A Connection with no pool behind it has no next query to poison, so it is left alone."
+ [^Connection conn]
+ (when (instance? C3P0ProxyConnection conn)
+ (try
+ (.rawConnectionOperation ^C3P0ProxyConnection conn
+ raw-connection-close-method
+ C3P0ProxyConnection/RAW_CONNECTION
+ (make-array Object 0))
+ (catch Throwable e
+ (log/debugf "Closing the raw connection to discard it failed: %s" (ex-message e))))
+ ;; Closing the raw Connection is invisible to c3p0: `rawConnectionOperation` reports nothing back to the pool, and
+ ;; the check-in reset only reads properties that a driver may answer from memory once closed — Hive's
+ ;; `getAutoCommit` is a bare `return true` — so on those drivers the pool would hand the dead Connection to the
+ ;; next query. Any *proxied* call routes its exception into c3p0, which then tests the physical Connection and
+ ;; destroys it when the test fails. `createStatement` is that call: JDBC requires it to throw on a closed
+ ;; Connection.
+ (try
+ (.close (.createStatement conn))
+ (catch Throwable _))))
diff --git a/src/metabase/driver/sql_jdbc/execute.clj b/src/metabase/driver/sql_jdbc/execute.clj
index c05325dccc0..006aee47b56 100644
--- a/src/metabase/driver/sql_jdbc/execute.clj
+++ b/src/metabase/driver/sql_jdbc/execute.clj
@@ -830,6 +830,41 @@
:pulse}]
(boolean (download-contexts context))))
+(def ^:private drivers-exempt-from-cancelation
+ "Drivers whose statements [[execute-reducible-query]] never cancels.
+ TODO: vertica is here only to find out whether it flakes because of cancelations. It should be removed afterwards!"
+ #{:vertica})
+
+(defmulti cancelation-poisons-connection?
+ "Whether canceling a Statement leaves this driver's Connection unfit for the next query, so that it must be thrown
+ away rather than returned to the pool.
+
+ Canceling abandons a result set the server is still producing, and the pool cannot clear what that leaves behind:
+ c3p0 resets `autoCommit`/`readOnly`/holdability on check-in, not driver-level wire state. A driver whose cancelation
+ leaves nothing pending on the wire should keep the default of `false`."
+ {:added "0.64.0", :arglists '([driver])}
+ driver/dispatch-on-initialized-driver
+ :hierarchy #'driver/hierarchy)
+
+(defmethod cancelation-poisons-connection? :default
+ [_driver]
+ false)
+
+(defn- cancel-statement!
+ "Cancel `stmt` on the DBMS side. Returns whether a cancelation was actually issued."
+ [driver ^Statement stmt]
+ (try
+ (when-not (.isClosed stmt)
+ (.cancel stmt)
+ true)
+ (catch SQLFeatureNotSupportedException _
+ (log/warnf "Statemet's `.cancel` method is not supported by the `%s` driver."
+ (name driver))
+ false)
+ (catch Throwable e
+ (log/infof "Statement cancelation failed: %s" (ex-message e))
+ false)))
+
(defn execute-reducible-query
"Default impl of [[metabase.driver/execute-reducible-query]] for sql-jdbc drivers."
{:added "0.35.0", :arglists '([driver query context respond])}
@@ -850,37 +885,52 @@
:stream? (or (download? (-> outer-query :info :context))
(= :table-rows-sample (-> outer-query :info :context)))}
(fn [^Connection conn]
- (with-open [stmt (statement-or-prepared-statement driver conn sql params (driver-api/canceled-chan))
- ^ResultSet rs (try
- (execute-statement-or-prepared-statement! driver stmt max-rows params sql)
- (catch Throwable e
- (throw (ex-info (tru "Error executing query: {0}" (ex-message e))
- (cond-> {:driver driver
- :sql (str/split-lines (driver/prettify-native-form driver sql))
- :params params
- :type driver-api/qp.error-type.invalid-query}
- (driver/query-canceled? driver e)
- (assoc :query/query-canceled? true))
- e))))]
- (let [rsmeta (.getMetaData rs)
- results-metadata {:cols (column-metadata driver rsmeta)}]
- (try (respond results-metadata (reducible-rows driver rs rsmeta (driver-api/canceled-chan)))
- ;; Following cancels the statement on the dbms side.
- ;; It avoids blocking `.close` call, in case we reduced the results subset eg. by means of
- ;; [[metabase.query-processor.middleware.limit/limit-xform]] middleware, while statement is still
- ;; in progress. This problem was encountered on Redshift. For details see the issue #39018.
- ;; It also handles situation where query is canceled through [[driver-api/canceled-chan]] (#41448).
- (finally
- ;; TODO: Following `when` is in place just to find out if vertica is flaking because of cancelations.
- ;; It should be removed afterwards!
- (when-not (= :vertica driver)
- (try (when-not (.isClosed stmt)
- (.cancel stmt))
- (catch SQLFeatureNotSupportedException _
- (log/warnf "Statemet's `.cancel` method is not supported by the `%s` driver."
- (name driver)))
- (catch Throwable e
- (log/infof "Statement cancelation failed: %s" (ex-message e))))))))))))))
+ ;; whether a cancelation was issued is only knowable inside the Statement's scope, but the Connection can
+ ;; only be dealt with once the Statement and ResultSet are closed, so it is carried out past both
+ (let [canceled? (volatile! false)]
+ (try
+ (with-open [stmt (statement-or-prepared-statement driver conn sql params (driver-api/canceled-chan))
+ ^ResultSet rs (try
+ (execute-statement-or-prepared-statement! driver stmt max-rows params sql)
+ (catch Throwable e
+ (throw (ex-info (tru "Error executing query: {0}" (ex-message e))
+ (cond-> {:driver driver
+ :sql (str/split-lines (driver/prettify-native-form driver sql))
+ :params params
+ :type driver-api/qp.error-type.invalid-query}
+ (driver/query-canceled? driver e)
+ (assoc :query/query-canceled? true))
+ e))))]
+ (let [rsmeta (.getMetaData rs)
+ results-metadata {:cols (column-metadata driver rsmeta)}
+ ;; whether the ResultSet ran out of rows is only observable while reducing, but it is needed after
+ ;; reduction has finished, so the row thunk records it on the way past
+ exhausted? (volatile! false)
+ next-row (row-thunk driver rs rsmeta)
+ rows (driver-api/reducible-rows
+ (fn []
+ (let [row (next-row)]
+ (when-not row
+ (vreset! exhausted? true))
+ row))
+ (driver-api/canceled-chan))]
+ (try (respond results-metadata rows)
+ ;; Following cancels the statement on the dbms side.
+ ;; It avoids blocking `.close` call, in case we reduced the results subset eg. by means of
+ ;; [[metabase.query-processor.middleware.limit/limit-xform]] middleware, while statement is still
+ ;; in progress. This problem was encountered on Redshift. For details see the issue #39018.
+ ;; It also handles situation where query is canceled through [[driver-api/canceled-chan]] (#41448).
+ ;; An exhausted ResultSet has nothing left to cancel.
+ (finally
+ (when-not (or @exhausted? (drivers-exempt-from-cancelation driver))
+ (vreset! canceled? (cancel-statement! driver stmt)))))))
+ ;; `with-open` has closed the ResultSet and Statement by the time this runs, which is required: on
+ ;; ClickHouse and Presto their `.close` round-trips to the server and throws once the Connection is gone.
+ ;; Discarding is safe here because this Connection was acquired for this query alone -- see
+ ;; [[do-with-resolved-connection]] for the cases where a Connection instead belongs to the caller.
+ (finally
+ (when (and @canceled? (cancelation-poisons-connection? driver))
+ (sql-jdbc.conn/discard-pooled-connection! conn))))))))))
(defn reducible-query
"Returns a reducible collection of rows as maps from `db` and a given SQL query. This is similar to [[jdbc/reducible-query]] but reuses the
diff --git a/test/metabase/driver/sql_jdbc/connection_test.clj b/test/metabase/driver/sql_jdbc/connection_test.clj
index c53cc1a8501..2dca17107fe 100644
--- a/test/metabase/driver/sql_jdbc/connection_test.clj
+++ b/test/metabase/driver/sql_jdbc/connection_test.clj
@@ -36,6 +36,8 @@
[next.jdbc :as next.jdbc]
[toucan2.core :as t2])
(:import
+ (com.mchange.v2.c3p0 C3P0ProxyConnection ComboPooledDataSource)
+ (java.sql DriverManager)
(org.h2.tools Server)))
(set! *warn-on-reflection* true)
@@ -823,3 +825,38 @@
(is (integer? (#'sql-jdbc.conn/default-ssh-tunnel-target-port driver/*driver*))))
(mt/test-drivers (mt/normal-driver-select {:+parent :sql-jdbc :-fns [has-default-port?]})
(is (nil? (#'sql-jdbc.conn/default-ssh-tunnel-target-port driver/*driver*)))))
+
+(def ^:private raw-connection-to-string-method
+ (.getMethod Object "toString" (make-array Class 0)))
+
+(defn- raw-connection-identity
+ "Identify the physical Connection behind a c3p0 proxy, to tell a recycled connection from a freshly acquired one."
+ [^C3P0ProxyConnection conn]
+ (.rawConnectionOperation conn
+ raw-connection-to-string-method
+ C3P0ProxyConnection/RAW_CONNECTION
+ (make-array Object 0)))
+
+(deftest discard-pooled-connection-test
+ (testing "discarding destroys the physical Connection, so the pool acquires a fresh one rather than recycling it"
+ (with-open [ds (doto (ComboPooledDataSource.)
+ (.setJdbcUrl "jdbc:h2:mem:discard-pooled-connection-test;DB_CLOSE_DELAY=-1")
+ (.setInitialPoolSize 1)
+ (.setMinPoolSize 1)
+ (.setMaxPoolSize 1))]
+ (letfn [(checked-out-identity []
+ (with-open [conn (.getConnection ds)]
+ (raw-connection-identity conn)))]
+ (let [before (checked-out-identity)]
+ (testing "a plain check-in/check-out cycle hands back the same physical Connection"
+ (is (= before (checked-out-identity))))
+ (with-open [conn (.getConnection ds)]
+ (sql-jdbc.conn/discard-pooled-connection! conn))
+ (testing "after discarding, the next checkout is a different physical Connection"
+ (is (not= before (checked-out-identity)))))))))
+
+(deftest discard-pooled-connection-leaves-unpooled-connection-alone-test
+ (testing "a Connection with no pool behind it has no next query to poison, so it is left open"
+ (with-open [conn (DriverManager/getConnection "jdbc:h2:mem:discard-unpooled-test;DB_CLOSE_DELAY=-1")]
+ (sql-jdbc.conn/discard-pooled-connection! conn)
+ (is (not (.isClosed conn))))))
diff --git a/test/metabase/driver/sql_jdbc/execute_test.clj b/test/metabase/driver/sql_jdbc/execute_test.clj
index 2e7ce862638..25e030ee6db 100644
--- a/test/metabase/driver/sql_jdbc/execute_test.clj
+++ b/test/metabase/driver/sql_jdbc/execute_test.clj
@@ -7,6 +7,9 @@
[metabase.driver.connection :as driver.conn]
[metabase.driver.h2 :as h2]
[metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
+ [metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
+ [metabase.query-processor :as qp]
[metabase.test :as mt]
[metabase.test.data.interface :as tx]
[metabase.util.malli.registry :as mr])
@@ -292,3 +295,33 @@
:native {:query "SELECT 1"}}
response (mt/user-http-request :crowberto :post 503 "dataset" query)]
(is (= "connection-pool-checkout-queue-full" (:error_type response))))))))))
+
+(defn- venues-rows
+ "Run an unaggregated venues query limited to `n` rows."
+ [n]
+ (let [mp (mt/metadata-provider)]
+ (-> (lib/query mp (lib.metadata/table mp (mt/id :venues)))
+ (lib/limit n)
+ qp/process-query
+ mt/rows)))
+
+(deftest cancel-statement-only-when-rows-remain-test
+ (testing "the statement is canceled only when reduction stopped before the ResultSet ran out of rows"
+ (mt/test-drivers (mt/normal-driver-select
+ {:+parent :sql-jdbc
+ :-fns [#'sql-jdbc.execute/drivers-exempt-from-cancelation]})
+ ;; take the dataset creation and sync queries before anything is counted
+ (venues-rows 1)
+ (let [cancels (atom 0)]
+ ;; the stub reports that no cancelation was issued, so nothing downstream acts on one
+ (mt/with-dynamic-fn-redefs [sql-jdbc.execute/cancel-statement! (fn [_driver _stmt]
+ (swap! cancels inc)
+ false)]
+ (testing "rows ran out, so there is nothing left to cancel"
+ (reset! cancels 0)
+ (is (= 100 (count (venues-rows 1000))))
+ (is (zero? @cancels)))
+ (testing "reduction stopped at the row limit while the statement was still producing (#39018)"
+ (reset! cancels 0)
+ (is (= 4 (count (venues-rows 4))))
+ (is (pos? @cancels))))))))