The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
GHY-3266 cancel slow data-warehouse queries via *canceled-chan* timer (#74776)
Eric Normand94f3c633ceb1
diff --git a/src/metabase/query_processor/setup.clj b/src/metabase/query_processor/setup.cljindex 963002f16cf..b2fc7ebc19f 100644--- a/src/metabase/query_processor/setup.clj+++ b/src/metabase/query_processor/setup.clj@@ -5,6 +5,7 @@[clojure.set :as set][metabase.config.core :as config][metabase.driver :as driver]+ [metabase.driver.settings :as driver.settings][metabase.driver.util :as driver.u][metabase.lib.computed :as lib.computed][metabase.lib.metadata :as lib.metadata]@@ -187,13 +188,32 @@(setting/with-database db(f query))))))+(defn- schedule-query-timeout-cancel!+ "Schedule a put of `::timeout` to `canceled-chan` after `*query-timeout-ms*`. Driver execution wires this channel+ to `Statement.cancel()` (see [[metabase.driver.sql-jdbc.execute/wire-up-canceled-chan-to-cancel-Statement!]]),+ which for MySQL/MariaDB issues `KILL QUERY` on a side connection — so this is the cross-driver path that actually+ stops a running query on the server when the timeout fires. `done-ch` closes when the query completes, releasing+ the go-block early. The timer never reads from `canceled-chan`, since callers (notably tests) may bind a regular+ channel where `alts!`/`<!` would consume the cancel signal away from the driver's listener."+ [canceled-chan done-ch]+ (let [timeout-ms (long driver.settings/*query-timeout-ms*)]+ (a/go+ (let [timeout-ch (a/timeout timeout-ms)+ [_ port] (a/alts! [done-ch timeout-ch])]+ (when (identical? port timeout-ch)+ (log/warnf "Query exceeded timeout of %d ms; canceling" timeout-ms)+ (a/put! canceled-chan ::timeout))))))+(mu/defn- do-with-canceled-chan :- fn?[f :- [:=> [:cat ::qp.schema/any-query] :any]](fn [query]- (if qp.pipeline/*canceled-chan*- (f query)- (binding [qp.pipeline/*canceled-chan* (a/promise-chan)]- (f query)))))+ (let [done-ch (a/promise-chan)]+ (try+ (binding [qp.pipeline/*canceled-chan* (or qp.pipeline/*canceled-chan* (a/promise-chan))]+ (schedule-query-timeout-cancel! qp.pipeline/*canceled-chan* done-ch)+ (f query))+ (finally+ (a/put! done-ch ::query-finished))))))(def ^:private setup-middleware"Setup middleware has the signaturediff --git a/test/metabase/driver/mysql_test.clj b/test/metabase/driver/mysql_test.cljindex 80e8c522696..0b7eb9201d0 100644--- a/test/metabase/driver/mysql_test.clj+++ b/test/metabase/driver/mysql_test.clj@@ -15,6 +15,7 @@[metabase.driver.mysql :as mysql][metabase.driver.mysql.actions :as mysql.actions][metabase.driver.mysql.ddl :as mysql.ddl]+ [metabase.driver.settings :as driver.settings][metabase.driver.sql-jdbc :as driver.sql-jdbc][metabase.driver.sql-jdbc.actions :as sql-jdbc.actions][metabase.driver.sql-jdbc.actions-test :as sql-jdbc.actions-test]@@ -1063,3 +1064,24 @@"webapp@localhost""SET ROLE 'webapp'@'localhost';")))))++(deftest ^:synchronized cancel-slow-mysql-query-via-query-timeout-test+ (mt/test-driver :mysql+ (testing "Slow MySQL query is cancelled server-side when *query-timeout-ms* elapses (GHY-3266)"+ ;; `SELECT SLEEP(60)` would normally take 60s. With `*query-timeout-ms*` of 2s the canceled-chan timer fires+ ;; (and `Statement.setQueryTimeout` also fires); both reach the MariaDB Connector/J `.cancel()` path which+ ;; issues `KILL QUERY` on a side connection. The query should error within a few seconds, not run to+ ;; completion. Cancellation evidence: the query throws AND finishes far short of 60s. We don't assert on the+ ;; exception message because QP output-schema middleware wraps the underlying SQLException — the timing is+ ;; the load-bearing assertion.+ (binding [driver.settings/*query-timeout-ms* 2000]+ (let [timer (u/start-timer)+ query (mt/native-query {:query "SELECT SLEEP(60) AS s"})+ result (try+ (qp/process-query query)+ (catch Throwable e e))+ elapsed (u/since-ms timer)]+ (is (instance? Throwable result)+ "query should throw rather than completing normally")+ (is (< elapsed 30000)+ (format "query should be cancelled well before SLEEP(60) completes naturally — took %.0f ms" elapsed)))))))diff --git a/test/metabase/query_processor/setup_test.clj b/test/metabase/query_processor/setup_test.cljindex 8054de5f92e..60774cdf924 100644--- a/test/metabase/query_processor/setup_test.clj+++ b/test/metabase/query_processor/setup_test.clj@@ -1,10 +1,77 @@(ns metabase.query-processor.setup-test(:require+ [clojure.core.async :as a][clojure.test :refer :all]+ [metabase.driver.settings :as driver.settings]+ [metabase.query-processor.pipeline :as qp.pipeline][metabase.query-processor.setup :as qp.setup]))+(set! *warn-on-reflection* true)+(deftest ^:parallel internal-query-type-test(testing "Make sure internal (audit app) queries work, even tho they don't have a :database ID."(qp.setup/with-qp-setup [query {:type :internal}](is (= {:type :internal}query)))))++(deftest ^:parallel canceled-chan-timer-fires-on-query-timeout-test+ (testing "When a query runs longer than *query-timeout-ms*, *canceled-chan* receives a ::timeout message"+ ;; this is the cross-driver cancel path that backs up Statement.setQueryTimeout — see GHY-3266 (MySQL slow view+ ;; fingerprinting): when setQueryTimeout doesn't reliably KILL QUERY on the server, the timer here puts+ ;; ::timeout on *canceled-chan*, which drives Statement.cancel() and forces the driver to issue a server-side+ ;; cancel anyway.+ (binding [driver.settings/*query-timeout-ms* 100]+ (let [observed-chan (promise)+ observed-val (promise)+ slow-f (fn [_query]+ (let [chan qp.pipeline/*canceled-chan*]+ (deliver observed-chan chan)+ (let [[v p] (a/alts!! [chan (a/timeout 2000)])]+ (deliver observed-val {:val v :from-canceled-chan? (identical? p chan)}))))+ f (#'qp.setup/do-with-canceled-chan slow-f)]+ (f {:type :internal})+ (is (some? @observed-chan)+ "the QP should bind a *canceled-chan* for the query")+ (is (:from-canceled-chan? @observed-val)+ "the canceled-chan should have been signaled before the 2s fallback timeout")+ (is (= ::qp.setup/timeout (:val @observed-val))+ "the timer should put ::timeout to *canceled-chan* once *query-timeout-ms* elapses")))))++(deftest ^:parallel canceled-chan-timer-does-not-fire-when-query-completes-first-test+ (testing "When a query finishes before *query-timeout-ms*, the timer never puts ::timeout on the chan"+ (binding [driver.settings/*query-timeout-ms* 60000]+ (let [observed-after-close (promise)+ fast-f (fn [_query]+ ;; mimic the pipeline's success path which closes *canceled-chan*+ (a/close! qp.pipeline/*canceled-chan*)+ ;; give the go-block a moment to see the close+ (Thread/sleep 100)+ (deliver observed-after-close (a/poll! qp.pipeline/*canceled-chan*)))+ f (#'qp.setup/do-with-canceled-chan fast-f)]+ (f {:type :internal})+ (is (nil? @observed-after-close)+ "a closed chan should poll nil — the timer must not have written ::timeout to it")))))++(deftest ^:parallel canceled-chan-timer-does-not-consume-external-cancel-signal-test+ (testing "Timer must not consume the cancel signal when callers bind a regular (a/chan)"+ ;; Regression for mongo `kill-an-in-flight-query-test`: that test binds `*canceled-chan*` to a regular+ ;; `(a/chan)` (not a promise-chan) and puts a cancel sentinel on it; the driver listener takes from the chan+ ;; to abort the in-flight query. If the timer reads from `*canceled-chan*` (via `alts!`/`<!`), it races the+ ;; driver listener for the value and on a regular chan one of them consumes it, breaking the other.+ (binding [driver.settings/*query-timeout-ms* 60000]+ (let [external-chan (a/chan)+ slow-f (fn [_query]+ (a/>!! qp.pipeline/*canceled-chan* ::external-cancel)+ ;; give the timer's go-block a moment to (incorrectly) consume the message+ (Thread/sleep 300))]+ (binding [qp.pipeline/*canceled-chan* external-chan]+ (let [f (#'qp.setup/do-with-canceled-chan slow-f)+ ;; consume from the external chan on a separate thread, mimicking what a driver's cancel listener+ ;; does. If the timer consumed the message, this take will time out.+ taken (future (a/alts!! [external-chan (a/timeout 1000)]))]+ (f {:type :internal})+ (let [[v p] (deref taken 2000 [::deref-timed-out ::deref-timed-out])]+ (is (identical? external-chan p)+ "the external listener should win the take, not time out")+ (is (= ::external-cancel v)+ "the external listener should see the original cancel sentinel"))))))))