Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Interrupt threads stuck after query cancellation (#69254)

Riley Thompson0464d028b3f0
diff --git a/src/metabase/server/settings.clj b/src/metabase/server/settings.clj
index 61f52e7c786..694c273003b 100644
--- a/src/metabase/server/settings.clj
+++ b/src/metabase/server/settings.clj
@@ -74,3 +74,17 @@ x.com")
:default false
:visibility :admin
:export? false)
+
+;; This is normally set via the env var `MB_THREAD_INTERRUPT_ESCALATION_TIMEOUT_MS`
+(defsetting thread-interrupt-escalation-timeout-ms
+ "By default, this is 0 and the thread interrupt escalation does not run."
+ :visibility :internal
+ :export? false
+ :type :integer
+ :default 0
+ :doc "Timeout in milliseconds to wait after query cancellation before escalating to thread interruption.
+ This is used to free up threads that are stuck waiting for a DB response after a query has been cancelled.")
+
+(def ^:dynamic ^Long *thread-interrupt-escalation-timeout-ms*
+ "Timeout in milliseconds to wait after query cancellation before escalating to thread interruption."
+ (thread-interrupt-escalation-timeout-ms))
diff --git a/src/metabase/server/streaming_response.clj b/src/metabase/server/streaming_response.clj
index 012b2d46574..f503780643d 100644
--- a/src/metabase/server/streaming_response.clj
+++ b/src/metabase/server/streaming_response.clj
@@ -23,6 +23,7 @@
(java.nio ByteBuffer)
(java.nio.channels ClosedChannelException SocketChannel)
(java.nio.charset StandardCharsets)
+ (java.util.concurrent Future)
(java.util.zip GZIPOutputStream)
(org.eclipse.jetty.ee9.nested Request)
(org.eclipse.jetty.io EofException SocketChannelEndPoint)))
@@ -88,6 +89,21 @@
(a/>!! canceled-chan ::thread-interrupted)
nil)))
+(defn- start-interrupt-escalation!
+ "If `*thread-interrupt-escalation-timeout-ms*` is set and we receive a cancellation,
+ then cancel the future (ie interrupt the thread) if the finished-chan doesn't complete before the timeout.
+ This is to handle JDBC drivers that deadlock on `(.cancel stmt)`."
+ [^Future fut finished-chan canceled-chan]
+ (when (pos? server.settings/*thread-interrupt-escalation-timeout-ms*)
+ (a/go
+ (when (a/<! canceled-chan)
+ (let [timeout-chan (a/timeout server.settings/*thread-interrupt-escalation-timeout-ms*)
+ [_ port] (a/alts! [finished-chan timeout-chan])]
+ (when (= port timeout-chan)
+ (log/infof "Task still running %s after cancellation, escalating to thread interruption"
+ (u/format-milliseconds server.settings/*thread-interrupt-escalation-timeout-ms*))
+ (.cancel fut true)))))))
+
(defn- do-f-async
"Runs `f` asynchronously on the streaming response `thread-pool`, returning immediately. When `f` finishes,
completes (i.e., closes) Jetty `async-context`."
@@ -101,13 +117,17 @@
(a/>!! finished-chan :unexpected-error)
(write-error! os e nil))
(finally
+ ;; Clear the interrupted flag to prevent the thread from
+ ;; carrying stale interrupted state to the next task.
+ (Thread/interrupted)
(a/>!! finished-chan (if (a/poll! canceled-chan)
:canceled
:completed))
(a/close! finished-chan)
(a/close! canceled-chan)
- (.complete async-context))))]
- (.submit (thread-pool/thread-pool) ^Runnable task)
+ (.complete async-context))))
+ fut (.submit (thread-pool/thread-pool) ^Runnable task)]
+ (start-interrupt-escalation! fut finished-chan canceled-chan)
nil))
;; `ring.middleware.gzip` doesn't work on our StreamingResponse class.
diff --git a/test/metabase/server/streaming_response_test.clj b/test/metabase/server/streaming_response_test.clj
index 8f3bf247085..e38c399ec5b 100644
--- a/test/metabase/server/streaming_response_test.clj
+++ b/test/metabase/server/streaming_response_test.clj
@@ -9,6 +9,7 @@
[metabase.query-processor.pipeline :as qp.pipeline]
[metabase.server.instance :as server.instance]
[metabase.server.protocols :as server.protocols]
+ [metabase.server.settings :as server.settings]
[metabase.server.streaming-response :as streaming-response]
[metabase.server.streaming-response.thread-pool :as thread-pool]
[metabase.test :as mt]
@@ -20,7 +21,7 @@
(jakarta.servlet AsyncContext ServletOutputStream)
(jakarta.servlet.http HttpServletResponse)
(java.io Closeable InputStream)
- (java.util.concurrent Executors)
+ (java.util.concurrent Executors Future)
(org.apache.commons.lang3.concurrent BasicThreadFactory$Builder)))
(set! *warn-on-reflection* true)
@@ -343,3 +344,87 @@
(is (not (mr/validate schema {:data "not a streaming response"})))
(is (= ["Non-streaming response returned from streaming endpoint"]
(me/humanize (mr/explain schema {:data "not a streaming response"}))))))))
+
+(deftest start-interrupt-escalation-cancels-future-test
+ (testing ".cancel is called when the task is still running after the interruption timeout"
+ (with-redefs [server.settings/*thread-interrupt-escalation-timeout-ms* 100]
+ (let [cancel-called? (promise)
+ mock-future (reify Future
+ (cancel [_ interrupt?]
+ (deliver cancel-called? interrupt?)
+ true)
+ (isCancelled [_] false)
+ (isDone [_] false)
+ (get [_] nil))
+ finished-chan (a/promise-chan)
+ canceled-chan (a/promise-chan)]
+ (#'streaming-response/start-interrupt-escalation! mock-future finished-chan canceled-chan)
+ (a/>!! canceled-chan ::request-canceled)
+ (let [result (deref cancel-called? 2000 ::timed-out)]
+ (is (true? result)))))))
+
+(deftest start-interrupt-escalation-no-cancel-when-task-finishes-test
+ (testing ".cancel is not called when the task finishes before the escalation timeout"
+ (with-redefs [server.settings/*thread-interrupt-escalation-timeout-ms* 2000]
+ (let [cancel-called? (promise)
+ mock-future (reify Future
+ (cancel [_ interrupt?]
+ (deliver cancel-called? interrupt?)
+ true)
+ (isCancelled [_] false)
+ (isDone [_] false)
+ (get [_] nil))
+ finished-chan (a/promise-chan)
+ canceled-chan (a/promise-chan)]
+ (#'streaming-response/start-interrupt-escalation! mock-future finished-chan canceled-chan)
+ (a/>!! canceled-chan ::request-canceled)
+ (Thread/sleep 50)
+ (a/>!! finished-chan :completed)
+ (is (= ::not-called (deref cancel-called? 200 ::not-called)))))))
+
+(deftest start-interrupt-escalation-noop-when-disabled-test
+ (testing "no-op when interrupt-escalation-timeout-ms is 0"
+ (with-redefs [server.settings/*thread-interrupt-escalation-timeout-ms* 0]
+ (let [cancel-called? (promise)
+ mock-future (reify Future
+ (cancel [_ interrupt?]
+ (deliver cancel-called? interrupt?)
+ true)
+ (isCancelled [_] false)
+ (isDone [_] false)
+ (get [_] nil))
+ finished-chan (a/promise-chan)
+ canceled-chan (a/promise-chan)]
+ (#'streaming-response/start-interrupt-escalation! mock-future finished-chan canceled-chan)
+ (a/>!! canceled-chan ::request-canceled)
+ (is (= ::not-called (deref cancel-called? 200 ::not-called)))))))
+
+(deftest interrupt-escalation-integration-test
+ (testing "A stuck streaming response thread is interrupted via escalation after cancellation"
+ (with-redefs [server.settings/*thread-interrupt-escalation-timeout-ms* 200]
+ (with-streaming-response-thread-pool!
+ (let [interrupted? (promise)
+ task-started? (promise)
+ complete-promise (promise)
+ finished-chan (a/promise-chan)
+ canceled-chan (a/promise-chan)]
+ (with-open [os (java.io.ByteArrayOutputStream.)]
+ (#'streaming-response/do-f-async
+ (reify AsyncContext
+ (complete [_]
+ (deliver complete-promise true)))
+ (fn [_os _canceled-chan]
+ (deliver task-started? true)
+ (try
+ ;; Simulate a stuck JDBC driver that blocks indefinitely
+ (Thread/sleep 60000)
+ (catch InterruptedException _e
+ (deliver interrupted? true))))
+ os
+ finished-chan
+ canceled-chan)
+ (is (true? (deref task-started? 5000 ::timed-out)))
+ (a/>!! canceled-chan ::request-canceled)
+ (is (true? (deref interrupted? 5000 ::timed-out)))
+ (is (true? (deref complete-promise 5000 ::timed-out)))
+ (is (= :canceled (a/poll! finished-chan)))))))))