Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

GHY-3266 cancel slow data-warehouse queries via *canceled-chan* timer (#74776)

Eric Normand94f3c633ceb1
diff --git a/src/metabase/query_processor/setup.clj b/src/metabase/query_processor/setup.clj
index 963002f16cf..b2fc7ebc19f 100644
--- a/src/metabase/query_processor/setup.clj
+++ b/src/metabase/query_processor/setup.clj
@@ -5,6 +5,7 @@
[clojure.set :as set]
[metabase.config.core :as config]
[metabase.driver :as driver]
+ [metabase.driver.settings :as driver.settings]
[metabase.driver.util :as driver.u]
[metabase.lib.computed :as lib.computed]
[metabase.lib.metadata :as lib.metadata]
@@ -187,13 +188,32 @@
(setting/with-database db
(f query))))))
+(defn- schedule-query-timeout-cancel!
+ "Schedule a put of `::timeout` to `canceled-chan` after `*query-timeout-ms*`. Driver execution wires this channel
+ to `Statement.cancel()` (see [[metabase.driver.sql-jdbc.execute/wire-up-canceled-chan-to-cancel-Statement!]]),
+ which for MySQL/MariaDB issues `KILL QUERY` on a side connection — so this is the cross-driver path that actually
+ stops a running query on the server when the timeout fires. `done-ch` closes when the query completes, releasing
+ the go-block early. The timer never reads from `canceled-chan`, since callers (notably tests) may bind a regular
+ channel where `alts!`/`<!` would consume the cancel signal away from the driver's listener."
+ [canceled-chan done-ch]
+ (let [timeout-ms (long driver.settings/*query-timeout-ms*)]
+ (a/go
+ (let [timeout-ch (a/timeout timeout-ms)
+ [_ port] (a/alts! [done-ch timeout-ch])]
+ (when (identical? port timeout-ch)
+ (log/warnf "Query exceeded timeout of %d ms; canceling" timeout-ms)
+ (a/put! canceled-chan ::timeout))))))
+
(mu/defn- do-with-canceled-chan :- fn?
[f :- [:=> [:cat ::qp.schema/any-query] :any]]
(fn [query]
- (if qp.pipeline/*canceled-chan*
- (f query)
- (binding [qp.pipeline/*canceled-chan* (a/promise-chan)]
- (f query)))))
+ (let [done-ch (a/promise-chan)]
+ (try
+ (binding [qp.pipeline/*canceled-chan* (or qp.pipeline/*canceled-chan* (a/promise-chan))]
+ (schedule-query-timeout-cancel! qp.pipeline/*canceled-chan* done-ch)
+ (f query))
+ (finally
+ (a/put! done-ch ::query-finished))))))
(def ^:private setup-middleware
"Setup middleware has the signature
diff --git a/test/metabase/driver/mysql_test.clj b/test/metabase/driver/mysql_test.clj
index 80e8c522696..0b7eb9201d0 100644
--- a/test/metabase/driver/mysql_test.clj
+++ b/test/metabase/driver/mysql_test.clj
@@ -15,6 +15,7 @@
[metabase.driver.mysql :as mysql]
[metabase.driver.mysql.actions :as mysql.actions]
[metabase.driver.mysql.ddl :as mysql.ddl]
+ [metabase.driver.settings :as driver.settings]
[metabase.driver.sql-jdbc :as driver.sql-jdbc]
[metabase.driver.sql-jdbc.actions :as sql-jdbc.actions]
[metabase.driver.sql-jdbc.actions-test :as sql-jdbc.actions-test]
@@ -1063,3 +1064,24 @@
"webapp@localhost"
"SET ROLE 'webapp'@'localhost';")))))
+
+(deftest ^:synchronized cancel-slow-mysql-query-via-query-timeout-test
+ (mt/test-driver :mysql
+ (testing "Slow MySQL query is cancelled server-side when *query-timeout-ms* elapses (GHY-3266)"
+ ;; `SELECT SLEEP(60)` would normally take 60s. With `*query-timeout-ms*` of 2s the canceled-chan timer fires
+ ;; (and `Statement.setQueryTimeout` also fires); both reach the MariaDB Connector/J `.cancel()` path which
+ ;; issues `KILL QUERY` on a side connection. The query should error within a few seconds, not run to
+ ;; completion. Cancellation evidence: the query throws AND finishes far short of 60s. We don't assert on the
+ ;; exception message because QP output-schema middleware wraps the underlying SQLException — the timing is
+ ;; the load-bearing assertion.
+ (binding [driver.settings/*query-timeout-ms* 2000]
+ (let [timer (u/start-timer)
+ query (mt/native-query {:query "SELECT SLEEP(60) AS s"})
+ result (try
+ (qp/process-query query)
+ (catch Throwable e e))
+ elapsed (u/since-ms timer)]
+ (is (instance? Throwable result)
+ "query should throw rather than completing normally")
+ (is (< elapsed 30000)
+ (format "query should be cancelled well before SLEEP(60) completes naturally — took %.0f ms" elapsed)))))))
diff --git a/test/metabase/query_processor/setup_test.clj b/test/metabase/query_processor/setup_test.clj
index 8054de5f92e..60774cdf924 100644
--- a/test/metabase/query_processor/setup_test.clj
+++ b/test/metabase/query_processor/setup_test.clj
@@ -1,10 +1,77 @@
(ns metabase.query-processor.setup-test
(:require
+ [clojure.core.async :as a]
[clojure.test :refer :all]
+ [metabase.driver.settings :as driver.settings]
+ [metabase.query-processor.pipeline :as qp.pipeline]
[metabase.query-processor.setup :as qp.setup]))
+(set! *warn-on-reflection* true)
+
(deftest ^:parallel internal-query-type-test
(testing "Make sure internal (audit app) queries work, even tho they don't have a :database ID."
(qp.setup/with-qp-setup [query {:type :internal}]
(is (= {:type :internal}
query)))))
+
+(deftest ^:parallel canceled-chan-timer-fires-on-query-timeout-test
+ (testing "When a query runs longer than *query-timeout-ms*, *canceled-chan* receives a ::timeout message"
+ ;; this is the cross-driver cancel path that backs up Statement.setQueryTimeout — see GHY-3266 (MySQL slow view
+ ;; fingerprinting): when setQueryTimeout doesn't reliably KILL QUERY on the server, the timer here puts
+ ;; ::timeout on *canceled-chan*, which drives Statement.cancel() and forces the driver to issue a server-side
+ ;; cancel anyway.
+ (binding [driver.settings/*query-timeout-ms* 100]
+ (let [observed-chan (promise)
+ observed-val (promise)
+ slow-f (fn [_query]
+ (let [chan qp.pipeline/*canceled-chan*]
+ (deliver observed-chan chan)
+ (let [[v p] (a/alts!! [chan (a/timeout 2000)])]
+ (deliver observed-val {:val v :from-canceled-chan? (identical? p chan)}))))
+ f (#'qp.setup/do-with-canceled-chan slow-f)]
+ (f {:type :internal})
+ (is (some? @observed-chan)
+ "the QP should bind a *canceled-chan* for the query")
+ (is (:from-canceled-chan? @observed-val)
+ "the canceled-chan should have been signaled before the 2s fallback timeout")
+ (is (= ::qp.setup/timeout (:val @observed-val))
+ "the timer should put ::timeout to *canceled-chan* once *query-timeout-ms* elapses")))))
+
+(deftest ^:parallel canceled-chan-timer-does-not-fire-when-query-completes-first-test
+ (testing "When a query finishes before *query-timeout-ms*, the timer never puts ::timeout on the chan"
+ (binding [driver.settings/*query-timeout-ms* 60000]
+ (let [observed-after-close (promise)
+ fast-f (fn [_query]
+ ;; mimic the pipeline's success path which closes *canceled-chan*
+ (a/close! qp.pipeline/*canceled-chan*)
+ ;; give the go-block a moment to see the close
+ (Thread/sleep 100)
+ (deliver observed-after-close (a/poll! qp.pipeline/*canceled-chan*)))
+ f (#'qp.setup/do-with-canceled-chan fast-f)]
+ (f {:type :internal})
+ (is (nil? @observed-after-close)
+ "a closed chan should poll nil — the timer must not have written ::timeout to it")))))
+
+(deftest ^:parallel canceled-chan-timer-does-not-consume-external-cancel-signal-test
+ (testing "Timer must not consume the cancel signal when callers bind a regular (a/chan)"
+ ;; Regression for mongo `kill-an-in-flight-query-test`: that test binds `*canceled-chan*` to a regular
+ ;; `(a/chan)` (not a promise-chan) and puts a cancel sentinel on it; the driver listener takes from the chan
+ ;; to abort the in-flight query. If the timer reads from `*canceled-chan*` (via `alts!`/`<!`), it races the
+ ;; driver listener for the value and on a regular chan one of them consumes it, breaking the other.
+ (binding [driver.settings/*query-timeout-ms* 60000]
+ (let [external-chan (a/chan)
+ slow-f (fn [_query]
+ (a/>!! qp.pipeline/*canceled-chan* ::external-cancel)
+ ;; give the timer's go-block a moment to (incorrectly) consume the message
+ (Thread/sleep 300))]
+ (binding [qp.pipeline/*canceled-chan* external-chan]
+ (let [f (#'qp.setup/do-with-canceled-chan slow-f)
+ ;; consume from the external chan on a separate thread, mimicking what a driver's cancel listener
+ ;; does. If the timer consumed the message, this take will time out.
+ taken (future (a/alts!! [external-chan (a/timeout 1000)]))]
+ (f {:type :internal})
+ (let [[v p] (deref taken 2000 [::deref-timed-out ::deref-timed-out])]
+ (is (identical? external-chan p)
+ "the external listener should win the take, not time out")
+ (is (= ::external-cancel v)
+ "the external listener should see the original cancel sentinel"))))))))