The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Fix "The result set is closed" flaky reuse of connections (#80033)
Ben Grabowa8db179087eb
diff --git a/docs/developers-guide/driver-changelog.md b/docs/developers-guide/driver-changelog.mdindex 6b0a37a3766..c1f306e1ea3 100644--- a/docs/developers-guide/driver-changelog.md+++ b/docs/developers-guide/driver-changelog.md@@ -6,6 +6,12 @@ title: Driver interface changelog## Metabase 0.64.0+- `metabase.driver.sql-jdbc.execute/cancelation-poisons-connection?` `[driver]` -- whether canceling a `Statement`+ leaves the `Connection` unfit for the next query, so that it must be discarded rather than returned to the+ connection pool. The query processor cancels a `Statement` whenever reduction stops before the `ResultSet` runs out+ of rows, which abandons a result set the server is still producing. Defaults to `false`, because most drivers+ recover on their own; `:sqlserver` overrides it to `true`.+- `metabase.driver/validate-impersonated-query` `[driver query]` now has a `:sql-jdbc` implementation thatenforces, for every JDBC driver, that a connection-impersonated native query is a single statement.diff --git a/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj b/modules/drivers/sqlserver/src/metabase/driver/sqlserver.cljindex 44adb046591..b860ab085a4 100644--- a/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj+++ b/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj@@ -1047,6 +1047,13 @@(.close stmt)(throw e)))))+(defmethod sql-jdbc.execute/cancelation-poisons-connection? :sqlserver+ [_driver]+ ;; `.cancel` sends an out-of-band TDS attention packet. Its acknowledgement is not drained before the Connection is+ ;; checked back into the pool, and it surfaces later as `The result set is closed.` while an unrelated query is+ ;; reading rows on the recycled Connection.+ true)+(defmethod sql.qp/inline-value [:sqlserver LocalDate][_ ^LocalDate t];; datefromparts(year, month, day)diff --git a/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj b/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.cljindex 2bb59e61db4..799a51d04d6 100644--- a/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj+++ b/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj@@ -1229,3 +1229,18 @@hosts #(set (driver/connection-parameter-hosts :sqlserver %))](is (contains? (hosts (assoc details :additional-options "serverName=10.0.0.1")) "10.0.0.1"))(is (not (contains? (hosts details) "10.0.0.1"))))))++(deftest cancelation-poisons-connection-test+ (testing "discarding the Connection a query canceled on leaves the pool able to serve later queries (#39018)"+ (mt/test-driver :sqlserver+ (is (true? (sql-jdbc.execute/cancelation-poisons-connection? :sqlserver))+ "SQL Server does not recover from a cancelation on its own, so the Connection must not be recycled")+ (letfn [(rows [table n]+ (let [mp (mt/metadata-provider)]+ (cond-> (lib/query mp (lib.metadata/table mp (mt/id table)))+ n (lib/limit n)+ true (-> qp/process-query mt/rows))))]+ ;; stopping at the limit leaves the statement producing, which is what triggers the cancel-and-discard+ (is (= 4 (count (rows :venues 4))))+ (testing "and a later query reading every row still succeeds"+ (is (= 1000 (count (rows :checkins nil)))))))))diff --git a/src/metabase/driver/sql_jdbc/connection.clj b/src/metabase/driver/sql_jdbc/connection.cljindex 1e745ff609b..f41c9a18962 100644--- a/src/metabase/driver/sql_jdbc/connection.clj+++ b/src/metabase/driver/sql_jdbc/connection.clj@@ -21,7 +21,8 @@[metabase.util.performance :refer [get-in mapv select-keys]][potemkin :as p])(:import- (com.mchange.v2.c3p0 DataSources)+ (com.mchange.v2.c3p0 C3P0ProxyConnection DataSources)+ (java.sql Connection)(javax.sql DataSource)(org.apache.logging.log4j Level)))@@ -544,3 +545,36 @@(defmethod driver/connection-spec :sql-jdbc [_driver db](db->pooled-connection-spec db))++(def ^:private raw-connection-close-method+ ;; c3p0 exposes the pooled physical Connection only through `rawConnectionOperation`, which names the operation as a+ ;; `java.lang.reflect.Method`. `unwrap` is not an alternative: c3p0 delegates it to the driver, and Hive's throws.+ (.getMethod Connection "close" (make-array Class 0)))++(defn discard-pooled-connection!+ "Destroy the physical Connection behind a c3p0 proxy, so the pool acquires a fresh one rather than handing this one+ to the next query. Use for a Connection left in a state the pool cannot reset, such as after canceling a Statement+ on a driver where that leaves unread protocol state on the wire.++ Only the caller that owns `conn` may call this: the pool has no way to tell the difference between a Connection+ whose borrower is finished with it and one that is still in use.++ A Connection with no pool behind it has no next query to poison, so it is left alone."+ [^Connection conn]+ (when (instance? C3P0ProxyConnection conn)+ (try+ (.rawConnectionOperation ^C3P0ProxyConnection conn+ raw-connection-close-method+ C3P0ProxyConnection/RAW_CONNECTION+ (make-array Object 0))+ (catch Throwable e+ (log/debugf "Closing the raw connection to discard it failed: %s" (ex-message e))))+ ;; Closing the raw Connection is invisible to c3p0: `rawConnectionOperation` reports nothing back to the pool, and+ ;; the check-in reset only reads properties that a driver may answer from memory once closed — Hive's+ ;; `getAutoCommit` is a bare `return true` — so on those drivers the pool would hand the dead Connection to the+ ;; next query. Any *proxied* call routes its exception into c3p0, which then tests the physical Connection and+ ;; destroys it when the test fails. `createStatement` is that call: JDBC requires it to throw on a closed+ ;; Connection.+ (try+ (.close (.createStatement conn))+ (catch Throwable _))))diff --git a/src/metabase/driver/sql_jdbc/execute.clj b/src/metabase/driver/sql_jdbc/execute.cljindex c05325dccc0..006aee47b56 100644--- a/src/metabase/driver/sql_jdbc/execute.clj+++ b/src/metabase/driver/sql_jdbc/execute.clj@@ -830,6 +830,41 @@:pulse}](boolean (download-contexts context))))+(def ^:private drivers-exempt-from-cancelation+ "Drivers whose statements [[execute-reducible-query]] never cancels.+ TODO: vertica is here only to find out whether it flakes because of cancelations. It should be removed afterwards!"+ #{:vertica})++(defmulti cancelation-poisons-connection?+ "Whether canceling a Statement leaves this driver's Connection unfit for the next query, so that it must be thrown+ away rather than returned to the pool.++ Canceling abandons a result set the server is still producing, and the pool cannot clear what that leaves behind:+ c3p0 resets `autoCommit`/`readOnly`/holdability on check-in, not driver-level wire state. A driver whose cancelation+ leaves nothing pending on the wire should keep the default of `false`."+ {:added "0.64.0", :arglists '([driver])}+ driver/dispatch-on-initialized-driver+ :hierarchy #'driver/hierarchy)++(defmethod cancelation-poisons-connection? :default+ [_driver]+ false)++(defn- cancel-statement!+ "Cancel `stmt` on the DBMS side. Returns whether a cancelation was actually issued."+ [driver ^Statement stmt]+ (try+ (when-not (.isClosed stmt)+ (.cancel stmt)+ true)+ (catch SQLFeatureNotSupportedException _+ (log/warnf "Statemet's `.cancel` method is not supported by the `%s` driver."+ (name driver))+ false)+ (catch Throwable e+ (log/infof "Statement cancelation failed: %s" (ex-message e))+ false)))+(defn execute-reducible-query"Default impl of [[metabase.driver/execute-reducible-query]] for sql-jdbc drivers."{:added "0.35.0", :arglists '([driver query context respond])}@@ -850,37 +885,52 @@:stream? (or (download? (-> outer-query :info :context))(= :table-rows-sample (-> outer-query :info :context)))}(fn [^Connection conn]- (with-open [stmt (statement-or-prepared-statement driver conn sql params (driver-api/canceled-chan))- ^ResultSet rs (try- (execute-statement-or-prepared-statement! driver stmt max-rows params sql)- (catch Throwable e- (throw (ex-info (tru "Error executing query: {0}" (ex-message e))- (cond-> {:driver driver- :sql (str/split-lines (driver/prettify-native-form driver sql))- :params params- :type driver-api/qp.error-type.invalid-query}- (driver/query-canceled? driver e)- (assoc :query/query-canceled? true))- e))))]- (let [rsmeta (.getMetaData rs)- results-metadata {:cols (column-metadata driver rsmeta)}]- (try (respond results-metadata (reducible-rows driver rs rsmeta (driver-api/canceled-chan)))- ;; Following cancels the statement on the dbms side.- ;; It avoids blocking `.close` call, in case we reduced the results subset eg. by means of- ;; [[metabase.query-processor.middleware.limit/limit-xform]] middleware, while statement is still- ;; in progress. This problem was encountered on Redshift. For details see the issue #39018.- ;; It also handles situation where query is canceled through [[driver-api/canceled-chan]] (#41448).- (finally- ;; TODO: Following `when` is in place just to find out if vertica is flaking because of cancelations.- ;; It should be removed afterwards!- (when-not (= :vertica driver)- (try (when-not (.isClosed stmt)- (.cancel stmt))- (catch SQLFeatureNotSupportedException _- (log/warnf "Statemet's `.cancel` method is not supported by the `%s` driver."- (name driver)))- (catch Throwable e- (log/infof "Statement cancelation failed: %s" (ex-message e))))))))))))))+ ;; whether a cancelation was issued is only knowable inside the Statement's scope, but the Connection can+ ;; only be dealt with once the Statement and ResultSet are closed, so it is carried out past both+ (let [canceled? (volatile! false)]+ (try+ (with-open [stmt (statement-or-prepared-statement driver conn sql params (driver-api/canceled-chan))+ ^ResultSet rs (try+ (execute-statement-or-prepared-statement! driver stmt max-rows params sql)+ (catch Throwable e+ (throw (ex-info (tru "Error executing query: {0}" (ex-message e))+ (cond-> {:driver driver+ :sql (str/split-lines (driver/prettify-native-form driver sql))+ :params params+ :type driver-api/qp.error-type.invalid-query}+ (driver/query-canceled? driver e)+ (assoc :query/query-canceled? true))+ e))))]+ (let [rsmeta (.getMetaData rs)+ results-metadata {:cols (column-metadata driver rsmeta)}+ ;; whether the ResultSet ran out of rows is only observable while reducing, but it is needed after+ ;; reduction has finished, so the row thunk records it on the way past+ exhausted? (volatile! false)+ next-row (row-thunk driver rs rsmeta)+ rows (driver-api/reducible-rows+ (fn []+ (let [row (next-row)]+ (when-not row+ (vreset! exhausted? true))+ row))+ (driver-api/canceled-chan))]+ (try (respond results-metadata rows)+ ;; Following cancels the statement on the dbms side.+ ;; It avoids blocking `.close` call, in case we reduced the results subset eg. by means of+ ;; [[metabase.query-processor.middleware.limit/limit-xform]] middleware, while statement is still+ ;; in progress. This problem was encountered on Redshift. For details see the issue #39018.+ ;; It also handles situation where query is canceled through [[driver-api/canceled-chan]] (#41448).+ ;; An exhausted ResultSet has nothing left to cancel.+ (finally+ (when-not (or @exhausted? (drivers-exempt-from-cancelation driver))+ (vreset! canceled? (cancel-statement! driver stmt)))))))+ ;; `with-open` has closed the ResultSet and Statement by the time this runs, which is required: on+ ;; ClickHouse and Presto their `.close` round-trips to the server and throws once the Connection is gone.+ ;; Discarding is safe here because this Connection was acquired for this query alone -- see+ ;; [[do-with-resolved-connection]] for the cases where a Connection instead belongs to the caller.+ (finally+ (when (and @canceled? (cancelation-poisons-connection? driver))+ (sql-jdbc.conn/discard-pooled-connection! conn))))))))))(defn reducible-query"Returns a reducible collection of rows as maps from `db` and a given SQL query. This is similar to [[jdbc/reducible-query]] but reuses thediff --git a/test/metabase/driver/sql_jdbc/connection_test.clj b/test/metabase/driver/sql_jdbc/connection_test.cljindex c53cc1a8501..2dca17107fe 100644--- a/test/metabase/driver/sql_jdbc/connection_test.clj+++ b/test/metabase/driver/sql_jdbc/connection_test.clj@@ -36,6 +36,8 @@[next.jdbc :as next.jdbc][toucan2.core :as t2])(:import+ (com.mchange.v2.c3p0 C3P0ProxyConnection ComboPooledDataSource)+ (java.sql DriverManager)(org.h2.tools Server)))(set! *warn-on-reflection* true)@@ -823,3 +825,38 @@(is (integer? (#'sql-jdbc.conn/default-ssh-tunnel-target-port driver/*driver*))))(mt/test-drivers (mt/normal-driver-select {:+parent :sql-jdbc :-fns [has-default-port?]})(is (nil? (#'sql-jdbc.conn/default-ssh-tunnel-target-port driver/*driver*)))))++(def ^:private raw-connection-to-string-method+ (.getMethod Object "toString" (make-array Class 0)))++(defn- raw-connection-identity+ "Identify the physical Connection behind a c3p0 proxy, to tell a recycled connection from a freshly acquired one."+ [^C3P0ProxyConnection conn]+ (.rawConnectionOperation conn+ raw-connection-to-string-method+ C3P0ProxyConnection/RAW_CONNECTION+ (make-array Object 0)))++(deftest discard-pooled-connection-test+ (testing "discarding destroys the physical Connection, so the pool acquires a fresh one rather than recycling it"+ (with-open [ds (doto (ComboPooledDataSource.)+ (.setJdbcUrl "jdbc:h2:mem:discard-pooled-connection-test;DB_CLOSE_DELAY=-1")+ (.setInitialPoolSize 1)+ (.setMinPoolSize 1)+ (.setMaxPoolSize 1))]+ (letfn [(checked-out-identity []+ (with-open [conn (.getConnection ds)]+ (raw-connection-identity conn)))]+ (let [before (checked-out-identity)]+ (testing "a plain check-in/check-out cycle hands back the same physical Connection"+ (is (= before (checked-out-identity))))+ (with-open [conn (.getConnection ds)]+ (sql-jdbc.conn/discard-pooled-connection! conn))+ (testing "after discarding, the next checkout is a different physical Connection"+ (is (not= before (checked-out-identity)))))))))++(deftest discard-pooled-connection-leaves-unpooled-connection-alone-test+ (testing "a Connection with no pool behind it has no next query to poison, so it is left open"+ (with-open [conn (DriverManager/getConnection "jdbc:h2:mem:discard-unpooled-test;DB_CLOSE_DELAY=-1")]+ (sql-jdbc.conn/discard-pooled-connection! conn)+ (is (not (.isClosed conn))))))diff --git a/test/metabase/driver/sql_jdbc/execute_test.clj b/test/metabase/driver/sql_jdbc/execute_test.cljindex 2e7ce862638..25e030ee6db 100644--- a/test/metabase/driver/sql_jdbc/execute_test.clj+++ b/test/metabase/driver/sql_jdbc/execute_test.clj@@ -7,6 +7,9 @@[metabase.driver.connection :as driver.conn][metabase.driver.h2 :as h2][metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.query-processor :as qp][metabase.test :as mt][metabase.test.data.interface :as tx][metabase.util.malli.registry :as mr])@@ -292,3 +295,33 @@:native {:query "SELECT 1"}}response (mt/user-http-request :crowberto :post 503 "dataset" query)](is (= "connection-pool-checkout-queue-full" (:error_type response))))))))))++(defn- venues-rows+ "Run an unaggregated venues query limited to `n` rows."+ [n]+ (let [mp (mt/metadata-provider)]+ (-> (lib/query mp (lib.metadata/table mp (mt/id :venues)))+ (lib/limit n)+ qp/process-query+ mt/rows)))++(deftest cancel-statement-only-when-rows-remain-test+ (testing "the statement is canceled only when reduction stopped before the ResultSet ran out of rows"+ (mt/test-drivers (mt/normal-driver-select+ {:+parent :sql-jdbc+ :-fns [#'sql-jdbc.execute/drivers-exempt-from-cancelation]})+ ;; take the dataset creation and sync queries before anything is counted+ (venues-rows 1)+ (let [cancels (atom 0)]+ ;; the stub reports that no cancelation was issued, so nothing downstream acts on one+ (mt/with-dynamic-fn-redefs [sql-jdbc.execute/cancel-statement! (fn [_driver _stmt]+ (swap! cancels inc)+ false)]+ (testing "rows ran out, so there is nothing left to cancel"+ (reset! cancels 0)+ (is (= 100 (count (venues-rows 1000))))+ (is (zero? @cancels)))+ (testing "reduction stopped at the row limit while the statement was still producing (#39018)"+ (reset! cancels 0)+ (is (= 4 (count (venues-rows 4))))+ (is (pos? @cancels))))))))