The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Interrupt threads stuck after query cancellation (#69254)
Riley Thompson0464d028b3f0
diff --git a/src/metabase/server/settings.clj b/src/metabase/server/settings.cljindex 61f52e7c786..694c273003b 100644--- a/src/metabase/server/settings.clj+++ b/src/metabase/server/settings.clj@@ -74,3 +74,17 @@ x.com"):default false:visibility :admin:export? false)++;; This is normally set via the env var `MB_THREAD_INTERRUPT_ESCALATION_TIMEOUT_MS`+(defsetting thread-interrupt-escalation-timeout-ms+ "By default, this is 0 and the thread interrupt escalation does not run."+ :visibility :internal+ :export? false+ :type :integer+ :default 0+ :doc "Timeout in milliseconds to wait after query cancellation before escalating to thread interruption.+ This is used to free up threads that are stuck waiting for a DB response after a query has been cancelled.")++(def ^:dynamic ^Long *thread-interrupt-escalation-timeout-ms*+ "Timeout in milliseconds to wait after query cancellation before escalating to thread interruption."+ (thread-interrupt-escalation-timeout-ms))diff --git a/src/metabase/server/streaming_response.clj b/src/metabase/server/streaming_response.cljindex 012b2d46574..f503780643d 100644--- a/src/metabase/server/streaming_response.clj+++ b/src/metabase/server/streaming_response.clj@@ -23,6 +23,7 @@(java.nio ByteBuffer)(java.nio.channels ClosedChannelException SocketChannel)(java.nio.charset StandardCharsets)+ (java.util.concurrent Future)(java.util.zip GZIPOutputStream)(org.eclipse.jetty.ee9.nested Request)(org.eclipse.jetty.io EofException SocketChannelEndPoint)))@@ -88,6 +89,21 @@(a/>!! canceled-chan ::thread-interrupted)nil)))+(defn- start-interrupt-escalation!+ "If `*thread-interrupt-escalation-timeout-ms*` is set and we receive a cancellation,+ then cancel the future (ie interrupt the thread) if the finished-chan doesn't complete before the timeout.+ This is to handle JDBC drivers that deadlock on `(.cancel stmt)`."+ [^Future fut finished-chan canceled-chan]+ (when (pos? server.settings/*thread-interrupt-escalation-timeout-ms*)+ (a/go+ (when (a/<! canceled-chan)+ (let [timeout-chan (a/timeout server.settings/*thread-interrupt-escalation-timeout-ms*)+ [_ port] (a/alts! [finished-chan timeout-chan])]+ (when (= port timeout-chan)+ (log/infof "Task still running %s after cancellation, escalating to thread interruption"+ (u/format-milliseconds server.settings/*thread-interrupt-escalation-timeout-ms*))+ (.cancel fut true)))))))+(defn- do-f-async"Runs `f` asynchronously on the streaming response `thread-pool`, returning immediately. When `f` finishes,completes (i.e., closes) Jetty `async-context`."@@ -101,13 +117,17 @@(a/>!! finished-chan :unexpected-error)(write-error! os e nil))(finally+ ;; Clear the interrupted flag to prevent the thread from+ ;; carrying stale interrupted state to the next task.+ (Thread/interrupted)(a/>!! finished-chan (if (a/poll! canceled-chan):canceled:completed))(a/close! finished-chan)(a/close! canceled-chan)- (.complete async-context))))]- (.submit (thread-pool/thread-pool) ^Runnable task)+ (.complete async-context))))+ fut (.submit (thread-pool/thread-pool) ^Runnable task)]+ (start-interrupt-escalation! fut finished-chan canceled-chan)nil));; `ring.middleware.gzip` doesn't work on our StreamingResponse class.diff --git a/test/metabase/server/streaming_response_test.clj b/test/metabase/server/streaming_response_test.cljindex 8f3bf247085..e38c399ec5b 100644--- a/test/metabase/server/streaming_response_test.clj+++ b/test/metabase/server/streaming_response_test.clj@@ -9,6 +9,7 @@[metabase.query-processor.pipeline :as qp.pipeline][metabase.server.instance :as server.instance][metabase.server.protocols :as server.protocols]+ [metabase.server.settings :as server.settings][metabase.server.streaming-response :as streaming-response][metabase.server.streaming-response.thread-pool :as thread-pool][metabase.test :as mt]@@ -20,7 +21,7 @@(jakarta.servlet AsyncContext ServletOutputStream)(jakarta.servlet.http HttpServletResponse)(java.io Closeable InputStream)- (java.util.concurrent Executors)+ (java.util.concurrent Executors Future)(org.apache.commons.lang3.concurrent BasicThreadFactory$Builder)))(set! *warn-on-reflection* true)@@ -343,3 +344,87 @@(is (not (mr/validate schema {:data "not a streaming response"})))(is (= ["Non-streaming response returned from streaming endpoint"](me/humanize (mr/explain schema {:data "not a streaming response"}))))))))++(deftest start-interrupt-escalation-cancels-future-test+ (testing ".cancel is called when the task is still running after the interruption timeout"+ (with-redefs [server.settings/*thread-interrupt-escalation-timeout-ms* 100]+ (let [cancel-called? (promise)+ mock-future (reify Future+ (cancel [_ interrupt?]+ (deliver cancel-called? interrupt?)+ true)+ (isCancelled [_] false)+ (isDone [_] false)+ (get [_] nil))+ finished-chan (a/promise-chan)+ canceled-chan (a/promise-chan)]+ (#'streaming-response/start-interrupt-escalation! mock-future finished-chan canceled-chan)+ (a/>!! canceled-chan ::request-canceled)+ (let [result (deref cancel-called? 2000 ::timed-out)]+ (is (true? result)))))))++(deftest start-interrupt-escalation-no-cancel-when-task-finishes-test+ (testing ".cancel is not called when the task finishes before the escalation timeout"+ (with-redefs [server.settings/*thread-interrupt-escalation-timeout-ms* 2000]+ (let [cancel-called? (promise)+ mock-future (reify Future+ (cancel [_ interrupt?]+ (deliver cancel-called? interrupt?)+ true)+ (isCancelled [_] false)+ (isDone [_] false)+ (get [_] nil))+ finished-chan (a/promise-chan)+ canceled-chan (a/promise-chan)]+ (#'streaming-response/start-interrupt-escalation! mock-future finished-chan canceled-chan)+ (a/>!! canceled-chan ::request-canceled)+ (Thread/sleep 50)+ (a/>!! finished-chan :completed)+ (is (= ::not-called (deref cancel-called? 200 ::not-called)))))))++(deftest start-interrupt-escalation-noop-when-disabled-test+ (testing "no-op when interrupt-escalation-timeout-ms is 0"+ (with-redefs [server.settings/*thread-interrupt-escalation-timeout-ms* 0]+ (let [cancel-called? (promise)+ mock-future (reify Future+ (cancel [_ interrupt?]+ (deliver cancel-called? interrupt?)+ true)+ (isCancelled [_] false)+ (isDone [_] false)+ (get [_] nil))+ finished-chan (a/promise-chan)+ canceled-chan (a/promise-chan)]+ (#'streaming-response/start-interrupt-escalation! mock-future finished-chan canceled-chan)+ (a/>!! canceled-chan ::request-canceled)+ (is (= ::not-called (deref cancel-called? 200 ::not-called)))))))++(deftest interrupt-escalation-integration-test+ (testing "A stuck streaming response thread is interrupted via escalation after cancellation"+ (with-redefs [server.settings/*thread-interrupt-escalation-timeout-ms* 200]+ (with-streaming-response-thread-pool!+ (let [interrupted? (promise)+ task-started? (promise)+ complete-promise (promise)+ finished-chan (a/promise-chan)+ canceled-chan (a/promise-chan)]+ (with-open [os (java.io.ByteArrayOutputStream.)]+ (#'streaming-response/do-f-async+ (reify AsyncContext+ (complete [_]+ (deliver complete-promise true)))+ (fn [_os _canceled-chan]+ (deliver task-started? true)+ (try+ ;; Simulate a stuck JDBC driver that blocks indefinitely+ (Thread/sleep 60000)+ (catch InterruptedException _e+ (deliver interrupted? true))))+ os+ finished-chan+ canceled-chan)+ (is (true? (deref task-started? 5000 ::timed-out)))+ (a/>!! canceled-chan ::request-canceled)+ (is (true? (deref interrupted? 5000 ::timed-out)))+ (is (true? (deref complete-promise 5000 ::timed-out)))+ (is (= :canceled (a/poll! finished-chan)))))))))