Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

fix: handle recycled jetty http response objects (#70674)

Edward Paget48d1a7c36e8e
diff --git a/src/metabase/server/streaming_response.clj b/src/metabase/server/streaming_response.clj
index 064a166318e..69d87659891 100644
--- a/src/metabase/server/streaming_response.clj
+++ b/src/metabase/server/streaming_response.clj
@@ -16,7 +16,7 @@
[ring.util.jakarta.servlet :as servlet]
[ring.util.response :as response])
(:import
- (jakarta.servlet AsyncContext)
+ (jakarta.servlet AsyncContext AsyncEvent AsyncListener)
(jakarta.servlet.http HttpServletResponse)
(java.io BufferedWriter OutputStream OutputStreamWriter)
(java.net SocketException)
@@ -24,6 +24,7 @@
(java.nio.channels ClosedChannelException SocketChannel)
(java.nio.charset StandardCharsets)
(java.util.concurrent Future)
+ (java.util.concurrent.atomic AtomicBoolean)
(java.util.zip GZIPOutputStream)
(org.eclipse.jetty.ee9.nested Request)
(org.eclipse.jetty.io EofException SocketChannelEndPoint)))
@@ -55,6 +56,18 @@
[[set-content-type!]] to interact with it."
nil)
+(def ^:dynamic *completed?*
+ "An `AtomicBoolean` that is set to `true` when the async context has been completed,
+ either by the worker thread or by Jetty's timeout/error callbacks. When `true`, the
+ response object may have been recycled and must not be touched."
+ nil)
+
+(defn- async-context-completed?
+ "Returns true if the async context has already been completed (by timeout, error, or worker thread).
+ When true, the response object may have been recycled by Jetty and must not be touched."
+ []
+ (and *completed?* (.get ^AtomicBoolean *completed?*)))
+
(defn- assert-response-bound! []
(when-not *response*
(throw (ex-info "Cannot call response control functions outside of a streaming-response context"
@@ -68,64 +81,81 @@
(.isCommitted ^HttpServletResponse *response*))
(defn set-status!
- "Set the HTTP status code on the response. No-op if the response is already committed.
+ "Set the HTTP status code on the response. No-op if the response is already committed
+ or the async context has already been completed (response may be recycled).
Raises if called outside a `streaming-response` context."
[code]
(assert-response-bound!)
- (when-not (committed?)
+ (when-not (or (async-context-completed?)
+ (committed?))
(.setStatus ^HttpServletResponse *response* (int code))))
(defn set-header!
- "Set a header on the HTTP response. No-op if the response is already committed.
+ "Set a header on the HTTP response. No-op if the response is already committed
+ or the async context has already been completed (response may be recycled).
Raises if called outside a `streaming-response` context."
[name value]
(assert-response-bound!)
- (when-not (committed?)
+ (when-not (or (async-context-completed?)
+ (committed?))
(.setHeader ^HttpServletResponse *response* (str name) (str value))))
(defn set-content-type!
- "Set the Content-Type on the HTTP response. No-op if the response is already committed.
+ "Set the Content-Type on the HTTP response. No-op if the response is already committed
+ or the async context has already been completed (response may be recycled).
Raises if called outside a `streaming-response` context."
[ct]
(assert-response-bound!)
- (when-not (committed?)
+ (when-not (or (async-context-completed?)
+ (committed?))
(.setContentType ^HttpServletResponse *response* (str ct))))
+(defn- log-skipped-error! [obj]
+ (if (instance? Throwable obj)
+ (log/error obj "Async context already completed, cannot write error to client")
+ (log/errorf "Async context already completed, cannot write error to client: %s" obj)))
+
+(defn- sanitize-error-obj [obj export-format]
+ (-> (if (not= :api export-format)
+ (walk/prewalk (fn [x]
+ (if (map? x)
+ (apply dissoc x [:json_query :preprocessed])
+ x))
+ obj)
+ obj)
+ (dissoc :export-format)
+ (cond-> (server.settings/hide-stacktraces) (dissoc :stacktrace :trace :via))))
+
(defn write-error!
- "Write an error to the output stream, formatting it nicely. Closes output stream afterwards."
+ "Write an error to the output stream, formatting it nicely. Closes output stream afterwards.
+ No-op if the async context has already been completed (response and stream may be recycled)."
([os obj export-format]
(write-error! os obj export-format nil))
([^OutputStream os obj export-format status-code]
- (when (and *response* (not (committed?)))
- (set-status! (or status-code 500))
- (set-content-type! "application/json"))
(cond
- (some #(instance? % obj)
- [InterruptedException EofException])
+ (async-context-completed?)
+ (log-skipped-error! obj)
+
+ (some #(instance? % obj) [InterruptedException EofException])
(log/trace "Error is an InterruptedException or EofException, not writing to output stream")
(instance? Throwable obj)
(recur os (format-exception obj) export-format status-code)
:else
- (with-open [os os]
- (log/trace (u/pprint-to-str (list 'write-error! obj)))
- (try
- (let [obj (-> (if (not= :api export-format)
- (walk/prewalk
- (fn [x]
- (if (map? x)
- (apply dissoc x [:json_query :preprocessed])
- x))
- obj)
- obj)
- (dissoc :export-format)
- (cond-> (server.settings/hide-stacktraces) (dissoc :stacktrace :trace :via)))]
- (with-open [writer (BufferedWriter. (OutputStreamWriter. os StandardCharsets/UTF_8))]
- (json/encode-to obj writer {})))
- (catch EofException _)
- (catch Throwable e
- (log/error e "Error writing error to output stream" obj)))))))
+ (do
+ (when (and *response* (not (committed?)))
+ (set-status! (or status-code 500))
+ (set-content-type! "application/json"))
+ (with-open [os os]
+ (log/trace (u/pprint-to-str (list 'write-error! obj)))
+ (try
+ (let [obj (sanitize-error-obj obj export-format)]
+ (with-open [writer (BufferedWriter. (OutputStreamWriter. os StandardCharsets/UTF_8))]
+ (json/encode-to obj writer {})))
+ (catch EofException _)
+ (catch Throwable e
+ (log/error e "Error writing error to output stream" obj))))))))
(defn- do-f* [f ^OutputStream os _finished-chan canceled-chan]
(try
@@ -154,11 +184,13 @@
(defn- do-f-async
"Runs `f` asynchronously on the streaming response `thread-pool`, returning immediately. When `f` finishes,
- completes (i.e., closes) Jetty `async-context`."
- [^AsyncContext async-context response f ^OutputStream os finished-chan canceled-chan]
+ completes (i.e., closes) Jetty `async-context`. `completed?` is an `AtomicBoolean` used to coordinate with
+ Jetty's timeout/error callbacks so that only one path calls `.complete`."
+ [^AsyncContext async-context response f ^OutputStream os finished-chan canceled-chan ^AtomicBoolean completed?]
{:pre [(some? os)]}
(let [task (^:once fn* []
- (binding [*response* response]
+ (binding [*response* response
+ *completed?* completed?]
(try
(do-f* f os finished-chan canceled-chan)
(catch Throwable e
@@ -174,7 +206,8 @@
:completed))
(a/close! finished-chan)
(a/close! canceled-chan)
- (.complete async-context)))))
+ (when (.compareAndSet completed? false true)
+ (.complete async-context))))))
fut (.submit (thread-pool/thread-pool) ^Runnable task)]
(start-interrupt-escalation! fut finished-chan canceled-chan)
nil))
@@ -294,7 +327,20 @@
(defn- respond
[{:keys [^HttpServletResponse response ^AsyncContext async-context request-map response-map request]}
f {:keys [content-type status headers], :as _options} finished-chan]
- (let [canceled-chan (a/promise-chan)]
+ (let [canceled-chan (a/promise-chan)
+ completed? (AtomicBoolean. false)]
+ (.addListener async-context
+ (reify AsyncListener
+ (onTimeout [_ _event]
+ (log/warn "Jetty async context timed out, completing request")
+ (when (.compareAndSet completed? false true)
+ (.complete async-context)))
+ (onError [_ event]
+ (log/warn (.getThrowable ^AsyncEvent event) "Jetty async context error, completing request")
+ (when (.compareAndSet completed? false true)
+ (.complete async-context)))
+ (onComplete [_ _event])
+ (onStartAsync [_ _event])))
(try
(.setStatus response (or status 202))
(let [gzip? (should-gzip-response? request-map)
@@ -310,7 +356,7 @@
(let [output-stream-delay (output-stream-delay gzip? response)
delay-os (delay-output-stream output-stream-delay)]
(start-async-cancel-loop! request finished-chan canceled-chan)
- (do-f-async async-context response f delay-os finished-chan canceled-chan)))
+ (do-f-async async-context response f delay-os finished-chan canceled-chan completed?)))
(catch Throwable e
(log/error e "Unexpected exception in do-f-async")
(try
@@ -320,7 +366,8 @@
(a/>!! finished-chan :unexpected-error)
(a/close! finished-chan)
(a/close! canceled-chan)
- (.complete async-context)))))
+ (when (.compareAndSet completed? false true)
+ (.complete async-context))))))
(declare render)
diff --git a/test/metabase/query_processor/streaming_test.clj b/test/metabase/query_processor/streaming_test.clj
index 71096431122..83d51597a05 100644
--- a/test/metabase/query_processor/streaming_test.clj
+++ b/test/metabase/query_processor/streaming_test.clj
@@ -177,6 +177,7 @@
([byytes offset length]
(.write os ^bytes byytes offset length))))))
:async-context (reify AsyncContext
+ (addListener [_ _])
(complete [_]
(deliver complete-promise true)))})
(is (true?
diff --git a/test/metabase/server/streaming_response_test.clj b/test/metabase/server/streaming_response_test.clj
index 2e6ff5f63cc..c02a123c6b8 100644
--- a/test/metabase/server/streaming_response_test.clj
+++ b/test/metabase/server/streaming_response_test.clj
@@ -21,7 +21,8 @@
(jakarta.servlet AsyncContext ServletOutputStream)
(jakarta.servlet.http HttpServletResponse)
(java.io ByteArrayOutputStream Closeable InputStream)
- (java.util.concurrent Executors Future)
+ (java.util.concurrent CountDownLatch Executors Future TimeUnit)
+ (java.util.concurrent.atomic AtomicBoolean)
(org.apache.commons.lang3.concurrent BasicThreadFactory$Builder)))
(set! *warn-on-reflection* true)
@@ -223,6 +224,7 @@
([byytes offset length]
(.write os ^bytes byytes offset length))))))
:async-context (reify AsyncContext
+ (addListener [_ _])
(complete [_]
(deliver complete-promise true)))})
(is (true?
@@ -541,9 +543,77 @@
(deliver interrupted? true))))
os
finished-chan
- canceled-chan)
+ canceled-chan
+ (AtomicBoolean. false))
(is (true? (deref task-started? 5000 ::timed-out)))
(a/>!! canceled-chan ::request-canceled)
(is (true? (deref interrupted? 5000 ::timed-out)))
(is (true? (deref complete-promise 5000 ::timed-out)))
(is (= :canceled (a/poll! finished-chan)))))))))
+
+(deftest write-error-recycled-response-test
+ (testing "write-error! is a no-op when *completed?* is true (response may be recycled)"
+ (let [os (ByteArrayOutputStream.)]
+ (binding [streaming-response/*response*
+ (reify HttpServletResponse
+ (isCommitted [_] (throw (NullPointerException. "response recycled")))
+ (setStatus [_ _] (throw (NullPointerException. "response recycled")))
+ (setContentType [_ _] (throw (NullPointerException. "response recycled"))))
+ streaming-response/*completed?*
+ (AtomicBoolean. true)]
+ (streaming-response/write-error! os {:error "test error"} :api 500))
+ (is (zero? (.size os))
+ "Nothing should be written when async context is already completed"))))
+
+(deftest async-timeout-completes-context-once-test
+ (testing "Only one of timeout callback or worker thread should call .complete"
+ (let [complete-count (atom 0)
+ completed? (AtomicBoolean. false)
+ mock-context (reify AsyncContext
+ (complete [_]
+ (swap! complete-count inc)))]
+ (is (true? (.compareAndSet completed? false true))
+ "Timeout callback should win compareAndSet")
+ (.complete mock-context)
+ (is (false? (.compareAndSet completed? false true))
+ "Worker thread's compareAndSet should return false")
+ (is (= 1 @complete-count)
+ ".complete should only have been called once"))))
+
+(deftest async-timeout-integration-test
+ (testing "Jetty async timeout does not cause NPE when worker thread finishes later"
+ (with-streaming-response-thread-pool!
+ (let [complete-called (CountDownLatch. 1)
+ task-started (CountDownLatch. 1)
+ task-can-finish (CountDownLatch. 1)
+ finished-chan (a/promise-chan)
+ canceled-chan (a/promise-chan)
+ completed? (AtomicBoolean. false)]
+ (with-open [os (ByteArrayOutputStream.)]
+ (#'streaming-response/do-f-async
+ (reify AsyncContext
+ (complete [_]
+ (.countDown complete-called)))
+ (reify HttpServletResponse
+ (isCommitted [_] false)
+ (setStatus [_ _] (when (.get completed?)
+ (throw (NullPointerException. "recycled"))))
+ (setContentType [_ _] (when (.get completed?)
+ (throw (NullPointerException. "recycled")))))
+ (fn [_os _canceled-chan]
+ (.countDown task-started)
+ (.await task-can-finish 5 TimeUnit/SECONDS)
+ (throw (ex-info "Query failed" {})))
+ os
+ finished-chan
+ canceled-chan
+ completed?)
+ (is (true? (.await task-started 5 TimeUnit/SECONDS))
+ "Worker thread should start")
+ (.set completed? true)
+ (.countDown task-can-finish)
+ (testing "Wait for worker thread to reach finally block"
+ (is (some? (a/<!! (a/go (a/alt! finished-chan ([v] v)
+ (a/timeout 5000) ([_] ::timed-out)))))))
+ (is (false? (.await complete-called 100 TimeUnit/MILLISECONDS))
+ "Worker thread should not call .complete when timeout already completed the context"))))))