Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Set network timeout in connection options (#66861)

Riley Thompsona77b0ee1da00
diff --git a/src/metabase/driver/settings.clj b/src/metabase/driver/settings.clj
index 4be8b3d3cc7..6b5b0ae16a5 100644
--- a/src/metabase/driver/settings.clj
+++ b/src/metabase/driver/settings.clj
@@ -100,6 +100,17 @@
Adjusting the timeout does not impact Metabase’s frontend.
Please be aware that other services (like Nginx) may still drop long-running queries.")
+;; This is normally set via the env var `MB_JDBC_NETWORKOUT_TIMEOUT_MS`
+(defsetting jdbc-networkout-timeout-ms
+ "By default, this is 30 minutes."
+ :visibility :internal
+ :export? false
+ :type :integer
+ :default (max (if config/is-prod? 1800000 600000) (* 1000 60 (+ (db-query-timeout-minutes) 5)))
+ :doc "Timeout in milliseconds to wait for database operations to complete. This is used to free up threads that
+ are stuck waiting for a database response in a socket read. See the documentation for more details:
+ https://docs.oracle.com/javase/8/docs/api/java/sql/Connection.html#setNetworkTimeout-java.util.concurrent.Executor-int-")
+
(defsetting jdbc-data-warehouse-max-connection-pool-size
"Maximum size of the c3p0 connection pool."
:visibility :internal
@@ -118,6 +129,10 @@
"Maximum amount of time query is allowed to run, in ms."
(u/minutes->ms (db-query-timeout-minutes)))
+(def ^:dynamic ^Long *network-timeout-ms*
+ "Maximum amount of time to wait for a response from the database, in ms."
+ (jdbc-networkout-timeout-ms))
+
(def ^:dynamic *allow-testing-h2-connections*
"Whether to allow testing new H2 connections. Normally this is disabled, which effectively means you cannot create new
H2 databases from the API, but this flag is here to disable that behavior for syncing existing databases, or when
diff --git a/src/metabase/driver/sql_jdbc/execute.clj b/src/metabase/driver/sql_jdbc/execute.clj
index 4d2ceb56299..fc20c7e156d 100644
--- a/src/metabase/driver/sql_jdbc/execute.clj
+++ b/src/metabase/driver/sql_jdbc/execute.clj
@@ -45,6 +45,7 @@
OffsetDateTime
OffsetTime
ZonedDateTime)
+ (java.util.concurrent Executors)
(javax.sql DataSource)))
(set! *warn-on-reflection* true)
@@ -352,6 +353,9 @@
(with-open [conn ^Connection (get-conn)]
(f conn)))))))
+(defonce ^:private network-timeout-executor
+ (delay (Executors/newCachedThreadPool)))
+
(mu/defn set-default-connection-options!
"Part of the default implementation of [[do-with-connection-with-options]]: set options for a newly fetched
Connection."
@@ -411,6 +415,12 @@
(.setAutoCommit conn false)
(catch Throwable e
(log/debug e "Error setting connection autoCommit to false"))))
+ (try
+ ;; setNetworkTimeout sets Socket.setSoTimeout() which releases from blocked socker reads.
+ ;; This is necessary because .close() doesn't interrupt threads stuck in native socket reads
+ (.setNetworkTimeout conn @network-timeout-executor driver.settings/*network-timeout-ms*)
+ (catch Throwable e
+ (log/debug e "Error setting network timeout for connection")))
(try
(log/trace (pr-str '(.setHoldability conn ResultSet/CLOSE_CURSORS_AT_COMMIT)))
(.setHoldability conn ResultSet/CLOSE_CURSORS_AT_COMMIT)
diff --git a/test/metabase/driver/postgres_test.clj b/test/metabase/driver/postgres_test.clj
index d8644636109..2014ec1cc5c 100644
--- a/test/metabase/driver/postgres_test.clj
+++ b/test/metabase/driver/postgres_test.clj
@@ -15,6 +15,7 @@
[metabase.driver.common.table-rows-sample :as table-rows-sample]
[metabase.driver.postgres :as postgres]
[metabase.driver.postgres.actions :as postgres.actions]
+ [metabase.driver.settings :as driver.settings]
[metabase.driver.sql :as driver.sql]
[metabase.driver.sql-jdbc.actions :as sql-jdbc.actions]
[metabase.driver.sql-jdbc.actions-test :as sql-jdbc.actions-test]
@@ -2105,3 +2106,28 @@
(is (= [[1 "1" "10101010" "1101" "111000111"]
[2 "0" "00001111" "10101" "1001001"]]
(mt/rows (qp/process-query query)))))))))))
+
+(deftest set-network-timeout-test
+ (mt/test-driver :postgres
+ (testing "network hangs are interrupted after *network-timeout-ms*"
+ (binding [driver.settings/*network-timeout-ms* 3000]
+ (is (thrown-with-msg?
+ org.postgresql.util.PSQLException
+ #"An I/O error occurred while sending to the backend"
+ (try
+ (sql-jdbc.execute/do-with-connection-with-options
+ driver/*driver* (mt/id) nil
+ (fn [^Connection conn]
+ (with-open [stmt (.createStatement conn)]
+ (.execute stmt "SELECT pg_sleep(6)"))))
+ (catch Exception e
+ (is (true? (some #(instance? java.net.SocketTimeoutException %)
+ (u/full-exception-chain e))))
+ (throw e)))))))
+ (testing "network hangs are not interrupted before *network-timeout-ms*"
+ (is (true?
+ (sql-jdbc.execute/do-with-connection-with-options
+ driver/*driver* (mt/id) nil
+ (fn [^Connection conn]
+ (with-open [stmt (.createStatement conn)]
+ (.execute stmt "SELECT pg_sleep(6)")))))))))
diff --git a/test/metabase/driver/sql_jdbc/execute_test.clj b/test/metabase/driver/sql_jdbc/execute_test.clj
index 70712fd0327..128ffff02ca 100644
--- a/test/metabase/driver/sql_jdbc/execute_test.clj
+++ b/test/metabase/driver/sql_jdbc/execute_test.clj
@@ -89,7 +89,9 @@
(setTransactionIsolation [level]
(vswap! connection-option-calls conj [:setTransactionIsolation level]))
(setHoldability [holdability]
- (vswap! connection-option-calls conj [:setHoldability holdability])))]
+ (vswap! connection-option-calls conj [:setHoldability holdability]))
+ (setNetworkTimeout [executor timeout-ms]
+ (vswap! connection-option-calls conj [:setNetworkTimeout timeout-ms])))]
(with-redefs [sql-jdbc.execute/do-with-resolved-connection-data-source
(fn [driver db options]
(if (:keep-open? options)
@@ -118,7 +120,8 @@
(let [calls @connection-option-calls]
(is (some #(= [:setReadOnly true] %) calls))
(is (some #(= [:setAutoCommit true] %) calls))
- (is (some #(= (first %) :setHoldability) calls))))))))))))
+ (is (some #(= (first %) :setHoldability) calls))
+ (is (some #(= (first %) :setNetworkTimeout) calls))))))))))))
(deftest is-conn-open-test
(testing "is-conn-open with valid check"