The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
fix: handle recycled jetty http response objects (#70674)
Edward Paget48d1a7c36e8e
diff --git a/src/metabase/server/streaming_response.clj b/src/metabase/server/streaming_response.cljindex 064a166318e..69d87659891 100644--- a/src/metabase/server/streaming_response.clj+++ b/src/metabase/server/streaming_response.clj@@ -16,7 +16,7 @@[ring.util.jakarta.servlet :as servlet][ring.util.response :as response])(:import- (jakarta.servlet AsyncContext)+ (jakarta.servlet AsyncContext AsyncEvent AsyncListener)(jakarta.servlet.http HttpServletResponse)(java.io BufferedWriter OutputStream OutputStreamWriter)(java.net SocketException)@@ -24,6 +24,7 @@(java.nio.channels ClosedChannelException SocketChannel)(java.nio.charset StandardCharsets)(java.util.concurrent Future)+ (java.util.concurrent.atomic AtomicBoolean)(java.util.zip GZIPOutputStream)(org.eclipse.jetty.ee9.nested Request)(org.eclipse.jetty.io EofException SocketChannelEndPoint)))@@ -55,6 +56,18 @@[[set-content-type!]] to interact with it."nil)+(def ^:dynamic *completed?*+ "An `AtomicBoolean` that is set to `true` when the async context has been completed,+ either by the worker thread or by Jetty's timeout/error callbacks. When `true`, the+ response object may have been recycled and must not be touched."+ nil)++(defn- async-context-completed?+ "Returns true if the async context has already been completed (by timeout, error, or worker thread).+ When true, the response object may have been recycled by Jetty and must not be touched."+ []+ (and *completed?* (.get ^AtomicBoolean *completed?*)))+(defn- assert-response-bound! [](when-not *response*(throw (ex-info "Cannot call response control functions outside of a streaming-response context"@@ -68,64 +81,81 @@(.isCommitted ^HttpServletResponse *response*))(defn set-status!- "Set the HTTP status code on the response. No-op if the response is already committed.+ "Set the HTTP status code on the response. No-op if the response is already committed+ or the async context has already been completed (response may be recycled).Raises if called outside a `streaming-response` context."[code](assert-response-bound!)- (when-not (committed?)+ (when-not (or (async-context-completed?)+ (committed?))(.setStatus ^HttpServletResponse *response* (int code))))(defn set-header!- "Set a header on the HTTP response. No-op if the response is already committed.+ "Set a header on the HTTP response. No-op if the response is already committed+ or the async context has already been completed (response may be recycled).Raises if called outside a `streaming-response` context."[name value](assert-response-bound!)- (when-not (committed?)+ (when-not (or (async-context-completed?)+ (committed?))(.setHeader ^HttpServletResponse *response* (str name) (str value))))(defn set-content-type!- "Set the Content-Type on the HTTP response. No-op if the response is already committed.+ "Set the Content-Type on the HTTP response. No-op if the response is already committed+ or the async context has already been completed (response may be recycled).Raises if called outside a `streaming-response` context."[ct](assert-response-bound!)- (when-not (committed?)+ (when-not (or (async-context-completed?)+ (committed?))(.setContentType ^HttpServletResponse *response* (str ct))))+(defn- log-skipped-error! [obj]+ (if (instance? Throwable obj)+ (log/error obj "Async context already completed, cannot write error to client")+ (log/errorf "Async context already completed, cannot write error to client: %s" obj)))++(defn- sanitize-error-obj [obj export-format]+ (-> (if (not= :api export-format)+ (walk/prewalk (fn [x]+ (if (map? x)+ (apply dissoc x [:json_query :preprocessed])+ x))+ obj)+ obj)+ (dissoc :export-format)+ (cond-> (server.settings/hide-stacktraces) (dissoc :stacktrace :trace :via))))+(defn write-error!- "Write an error to the output stream, formatting it nicely. Closes output stream afterwards."+ "Write an error to the output stream, formatting it nicely. Closes output stream afterwards.+ No-op if the async context has already been completed (response and stream may be recycled)."([os obj export-format](write-error! os obj export-format nil))([^OutputStream os obj export-format status-code]- (when (and *response* (not (committed?)))- (set-status! (or status-code 500))- (set-content-type! "application/json"))(cond- (some #(instance? % obj)- [InterruptedException EofException])+ (async-context-completed?)+ (log-skipped-error! obj)++ (some #(instance? % obj) [InterruptedException EofException])(log/trace "Error is an InterruptedException or EofException, not writing to output stream")(instance? Throwable obj)(recur os (format-exception obj) export-format status-code):else- (with-open [os os]- (log/trace (u/pprint-to-str (list 'write-error! obj)))- (try- (let [obj (-> (if (not= :api export-format)- (walk/prewalk- (fn [x]- (if (map? x)- (apply dissoc x [:json_query :preprocessed])- x))- obj)- obj)- (dissoc :export-format)- (cond-> (server.settings/hide-stacktraces) (dissoc :stacktrace :trace :via)))]- (with-open [writer (BufferedWriter. (OutputStreamWriter. os StandardCharsets/UTF_8))]- (json/encode-to obj writer {})))- (catch EofException _)- (catch Throwable e- (log/error e "Error writing error to output stream" obj)))))))+ (do+ (when (and *response* (not (committed?)))+ (set-status! (or status-code 500))+ (set-content-type! "application/json"))+ (with-open [os os]+ (log/trace (u/pprint-to-str (list 'write-error! obj)))+ (try+ (let [obj (sanitize-error-obj obj export-format)]+ (with-open [writer (BufferedWriter. (OutputStreamWriter. os StandardCharsets/UTF_8))]+ (json/encode-to obj writer {})))+ (catch EofException _)+ (catch Throwable e+ (log/error e "Error writing error to output stream" obj))))))))(defn- do-f* [f ^OutputStream os _finished-chan canceled-chan](try@@ -154,11 +184,13 @@(defn- do-f-async"Runs `f` asynchronously on the streaming response `thread-pool`, returning immediately. When `f` finishes,- completes (i.e., closes) Jetty `async-context`."- [^AsyncContext async-context response f ^OutputStream os finished-chan canceled-chan]+ completes (i.e., closes) Jetty `async-context`. `completed?` is an `AtomicBoolean` used to coordinate with+ Jetty's timeout/error callbacks so that only one path calls `.complete`."+ [^AsyncContext async-context response f ^OutputStream os finished-chan canceled-chan ^AtomicBoolean completed?]{:pre [(some? os)]}(let [task (^:once fn* []- (binding [*response* response]+ (binding [*response* response+ *completed?* completed?](try(do-f* f os finished-chan canceled-chan)(catch Throwable e@@ -174,7 +206,8 @@:completed))(a/close! finished-chan)(a/close! canceled-chan)- (.complete async-context)))))+ (when (.compareAndSet completed? false true)+ (.complete async-context))))))fut (.submit (thread-pool/thread-pool) ^Runnable task)](start-interrupt-escalation! fut finished-chan canceled-chan)nil))@@ -294,7 +327,20 @@(defn- respond[{:keys [^HttpServletResponse response ^AsyncContext async-context request-map response-map request]}f {:keys [content-type status headers], :as _options} finished-chan]- (let [canceled-chan (a/promise-chan)]+ (let [canceled-chan (a/promise-chan)+ completed? (AtomicBoolean. false)]+ (.addListener async-context+ (reify AsyncListener+ (onTimeout [_ _event]+ (log/warn "Jetty async context timed out, completing request")+ (when (.compareAndSet completed? false true)+ (.complete async-context)))+ (onError [_ event]+ (log/warn (.getThrowable ^AsyncEvent event) "Jetty async context error, completing request")+ (when (.compareAndSet completed? false true)+ (.complete async-context)))+ (onComplete [_ _event])+ (onStartAsync [_ _event])))(try(.setStatus response (or status 202))(let [gzip? (should-gzip-response? request-map)@@ -310,7 +356,7 @@(let [output-stream-delay (output-stream-delay gzip? response)delay-os (delay-output-stream output-stream-delay)](start-async-cancel-loop! request finished-chan canceled-chan)- (do-f-async async-context response f delay-os finished-chan canceled-chan)))+ (do-f-async async-context response f delay-os finished-chan canceled-chan completed?)))(catch Throwable e(log/error e "Unexpected exception in do-f-async")(try@@ -320,7 +366,8 @@(a/>!! finished-chan :unexpected-error)(a/close! finished-chan)(a/close! canceled-chan)- (.complete async-context)))))+ (when (.compareAndSet completed? false true)+ (.complete async-context))))))(declare render)diff --git a/test/metabase/query_processor/streaming_test.clj b/test/metabase/query_processor/streaming_test.cljindex 71096431122..83d51597a05 100644--- a/test/metabase/query_processor/streaming_test.clj+++ b/test/metabase/query_processor/streaming_test.clj@@ -177,6 +177,7 @@([byytes offset length](.write os ^bytes byytes offset length)))))):async-context (reify AsyncContext+ (addListener [_ _])(complete [_](deliver complete-promise true)))})(is (true?diff --git a/test/metabase/server/streaming_response_test.clj b/test/metabase/server/streaming_response_test.cljindex 2e6ff5f63cc..c02a123c6b8 100644--- a/test/metabase/server/streaming_response_test.clj+++ b/test/metabase/server/streaming_response_test.clj@@ -21,7 +21,8 @@(jakarta.servlet AsyncContext ServletOutputStream)(jakarta.servlet.http HttpServletResponse)(java.io ByteArrayOutputStream Closeable InputStream)- (java.util.concurrent Executors Future)+ (java.util.concurrent CountDownLatch Executors Future TimeUnit)+ (java.util.concurrent.atomic AtomicBoolean)(org.apache.commons.lang3.concurrent BasicThreadFactory$Builder)))(set! *warn-on-reflection* true)@@ -223,6 +224,7 @@([byytes offset length](.write os ^bytes byytes offset length)))))):async-context (reify AsyncContext+ (addListener [_ _])(complete [_](deliver complete-promise true)))})(is (true?@@ -541,9 +543,77 @@(deliver interrupted? true))))osfinished-chan- canceled-chan)+ canceled-chan+ (AtomicBoolean. false))(is (true? (deref task-started? 5000 ::timed-out)))(a/>!! canceled-chan ::request-canceled)(is (true? (deref interrupted? 5000 ::timed-out)))(is (true? (deref complete-promise 5000 ::timed-out)))(is (= :canceled (a/poll! finished-chan)))))))))++(deftest write-error-recycled-response-test+ (testing "write-error! is a no-op when *completed?* is true (response may be recycled)"+ (let [os (ByteArrayOutputStream.)]+ (binding [streaming-response/*response*+ (reify HttpServletResponse+ (isCommitted [_] (throw (NullPointerException. "response recycled")))+ (setStatus [_ _] (throw (NullPointerException. "response recycled")))+ (setContentType [_ _] (throw (NullPointerException. "response recycled"))))+ streaming-response/*completed?*+ (AtomicBoolean. true)]+ (streaming-response/write-error! os {:error "test error"} :api 500))+ (is (zero? (.size os))+ "Nothing should be written when async context is already completed"))))++(deftest async-timeout-completes-context-once-test+ (testing "Only one of timeout callback or worker thread should call .complete"+ (let [complete-count (atom 0)+ completed? (AtomicBoolean. false)+ mock-context (reify AsyncContext+ (complete [_]+ (swap! complete-count inc)))]+ (is (true? (.compareAndSet completed? false true))+ "Timeout callback should win compareAndSet")+ (.complete mock-context)+ (is (false? (.compareAndSet completed? false true))+ "Worker thread's compareAndSet should return false")+ (is (= 1 @complete-count)+ ".complete should only have been called once"))))++(deftest async-timeout-integration-test+ (testing "Jetty async timeout does not cause NPE when worker thread finishes later"+ (with-streaming-response-thread-pool!+ (let [complete-called (CountDownLatch. 1)+ task-started (CountDownLatch. 1)+ task-can-finish (CountDownLatch. 1)+ finished-chan (a/promise-chan)+ canceled-chan (a/promise-chan)+ completed? (AtomicBoolean. false)]+ (with-open [os (ByteArrayOutputStream.)]+ (#'streaming-response/do-f-async+ (reify AsyncContext+ (complete [_]+ (.countDown complete-called)))+ (reify HttpServletResponse+ (isCommitted [_] false)+ (setStatus [_ _] (when (.get completed?)+ (throw (NullPointerException. "recycled"))))+ (setContentType [_ _] (when (.get completed?)+ (throw (NullPointerException. "recycled")))))+ (fn [_os _canceled-chan]+ (.countDown task-started)+ (.await task-can-finish 5 TimeUnit/SECONDS)+ (throw (ex-info "Query failed" {})))+ os+ finished-chan+ canceled-chan+ completed?)+ (is (true? (.await task-started 5 TimeUnit/SECONDS))+ "Worker thread should start")+ (.set completed? true)+ (.countDown task-can-finish)+ (testing "Wait for worker thread to reach finally block"+ (is (some? (a/<!! (a/go (a/alt! finished-chan ([v] v)+ (a/timeout 5000) ([_] ::timed-out)))))))+ (is (false? (.await complete-called 100 TimeUnit/MILLISECONDS))+ "Worker thread should not call .complete when timeout already completed the context"))))))