Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

request cancelation: this time for real (#65660)

Oleksandr Solovyov559d3357a2a3
diff --git a/enterprise/backend/src/metabase_enterprise/metabot_v3/client.clj b/enterprise/backend/src/metabase_enterprise/metabot_v3/client.clj
index 9951850328a..44bf8ed77c8 100644
--- a/enterprise/backend/src/metabase_enterprise/metabot_v3/client.clj
+++ b/enterprise/backend/src/metabase_enterprise/metabot_v3/client.clj
@@ -25,7 +25,7 @@
[metabase.util.malli.registry :as mr]
[metabase.util.o11y :refer [with-span]])
(:import
- (java.io BufferedReader)
+ (java.io BufferedReader Closeable FilterInputStream InputStream)
(org.eclipse.jetty.io EofException)))
(set! *warn-on-reflection* true)
@@ -108,6 +108,19 @@
(defn- generate-embeddings-endpoint []
(str (metabot-v3.settings/ai-service-base-url) "/v1/embeddings"))
+(defn- quick-closing-body
+ "Some requests come with body wrapped in ContentLengthInputStream, and that will never close the underlying stream.
+ So we just close the client itself.
+
+ Please use `Connection: close` header when using this function to prevent connection being reused by HttpClient.
+
+ See: https://github.com/dakrone/clj-http/issues/627
+ Also: metabase-enterprise.metabot-v3.api-test/closing-connection-test (for 'chunked')"
+ [response]
+ (proxy [FilterInputStream] [^InputStream (:body response)]
+ (close []
+ (.close ^Closeable (:http-client response)))))
+
(mu/defn streaming-request :- :any
"Make a streaming V2 request to the AI Service
@@ -149,10 +162,16 @@
"Content-Type" "application/json;charset=UTF-8"
"x-metabase-instance-token" (premium-features/premium-embedding-token)
"x-metabase-session-token" session-id
- "x-metabase-url" (system/site-url)}
+ "x-metabase-url" (system/site-url)
+ ;; close conn so it's not reused so that when we use
+ ;; `quick-closing-body` there are no weird problems
+ "Connection" "close"}
:body (->json-bytes body)
:throw-exceptions false
- :as :stream}
+ :as :stream
+ ;; Don't compress streaming responses - adds latency and breaks
+ ;; cancellation when streams are incomplete
+ :decompress-body false}
*debug* (assoc :debug true))
response (post! url options)
lines (when on-complete
@@ -162,13 +181,13 @@
canceled (atom nil)]
(metabot-v3.context/log (:body response) :llm.log/llm->be)
(log/debugf "Response from AI Proxy:\n%s" (u/pprint-to-str (select-keys response #{:body :status :headers})))
- (when-not (= (:status response) 200)
+ (when-not (#{200 202} (:status response))
(throw (ex-info (format "Error: unexpected status code: %d %s" (:status response) (:reason-phrase response))
{:request (assoc options :body body)
:response response})))
(sr/streaming-response {:content-type "text/event-stream; charset=utf-8"} [os canceled-chan]
- ;; exiting with-open will close underlying request
- (with-open [^BufferedReader response-reader (io/reader (:body response))]
+ ;; see `quick-closing-body` docs and see `Connection` header supporting this behavior
+ (with-open [^BufferedReader response-reader (io/reader (quick-closing-body response))]
;; Response from the AI Service will send response parts separated by newline
(loop [^String line (.readLine response-reader)]
(cond
@@ -186,8 +205,12 @@
(.write (.getBytes "\n"))
;; Immediately flush so it feels fluid on the frontend
(.flush))
- (catch EofException _ nil))
- (recur (.readLine response-reader))))))
+ (catch EofException _
+ (reset! canceled true)
+ (log/debug "Request cancelled, through exception")))
+ (when-not @canceled
+ ;; `recur` cannot be inside of `try`, so we have to signal stop somehow
+ (recur (.readLine response-reader)))))))
(when on-complete
(on-complete @lines))))
(catch Throwable e
diff --git a/enterprise/backend/test/metabase_enterprise/metabot_v3/api_test.clj b/enterprise/backend/test/metabase_enterprise/metabot_v3/api_test.clj
index 9cdf9bf2233..22dff4094a4 100644
--- a/enterprise/backend/test/metabase_enterprise/metabot_v3/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/metabot_v3/api_test.clj
@@ -1,14 +1,17 @@
(ns metabase-enterprise.metabot-v3.api-test
(:require
[clj-http.client :as http]
+ [clojure.core.async :as a]
[clojure.string :as str]
[clojure.test :refer :all]
+ [compojure.response]
[medley.core :as m]
[metabase-enterprise.metabot-v3.api :as api]
[metabase-enterprise.metabot-v3.client :as client]
[metabase-enterprise.metabot-v3.client-test :as client-test]
[metabase-enterprise.metabot-v3.util :as metabot.u]
[metabase.search.test-util :as search.tu]
+ [metabase.server.instance :as server.instance]
[metabase.server.streaming-response :as sr]
[metabase.test :as mt]
[metabase.util :as u]
@@ -65,36 +68,67 @@
messages))))))))))
(deftest closing-connection-test
- (let [mock-response (client-test/make-mock-stream-response
- (mapv #(str "msg-" % "\n") (range 30))
- {"some-model" {:prompt 12 :completion 3}})
- messages (atom nil)]
- (mt/test-helpers-set-global-values!
- (search.tu/with-index-disabled
- (mt/with-premium-features #{:metabot-v3}
- (with-redefs [client/post! (client-test/mock-post! mock-response {:delay-ms 5})
- api/store-message! (fn [_conv-id _prof-id msgs]
- (reset! messages msgs))
- sr/async-cancellation-poll-interval-ms 5]
- (testing "Closing body stream drops connection"
- (let [body (mt/user-real-request :rasta :post 202 "ee/metabot-v3/agent-streaming"
- {:request-options {:as :stream
- :decompress-body false}}
- {:message "Test closure"
- :context {}
- :conversation_id (str (random-uuid))
- :history []
- :state {}})]
- (.read ^java.io.InputStream body) ;; start the handler
- (.close ^java.io.Closeable body)
- (u/poll {:thunk #(= :assistant (:role (first @messages)))
- :done? true?
- :interval-ms 5})
- (is (= :assistant (:role (first @messages)))
- "store-messages! was called in the end on the lines streaming-request managed to write to OS")
- ;; if this flakes in CI, increase the number a bit; but it was 2 quite consistently for me
- (is (> 10 (-> @messages first :content str/split-lines count))
- "But we shouldn't go through all 30 of them")))))))))
+ (let [messages (atom nil)
+ cnt (atom 30)
+ canceled (atom nil)
+ ai-handler (fn [req respond _raise]
+ (respond
+ (compojure.response/render
+ (sr/streaming-response {:content-type "text/event-stream; charset=utf-8"
+ ;; Transfer-Encoding: chunked is what makes normal .close on body fail
+ ;; See: `metabase-enterprise.metabot-v3.client/quick-closing-body`
+ :headers {"Transfer-Encoding" "chunked"}} [os canceled-chan]
+ (try
+ (loop []
+ (if (a/poll! canceled-chan)
+ (reset! canceled :nice)
+ (do
+ (.write os (.getBytes (str "2:" (json/encode {:msg @cnt}) "\n")))
+ (.flush os)
+ (swap! cnt dec)
+ (Thread/sleep 10)
+ (when (pos? @cnt)
+ (recur)))))
+ (catch Exception _e
+ (reset! canceled :not-nice))))
+ req)))
+ ai-server (doto (server.instance/create-server ai-handler {:port 0 :join? false})
+ .start)
+ ai-url (str "http://localhost:" (.. ai-server getURI getPort))]
+ (try
+ (mt/test-helpers-set-global-values!
+ (search.tu/with-index-disabled
+ (mt/with-premium-features #{:metabot-v3}
+ (with-redefs [client/ai-url (constantly ai-url)
+ api/store-message! (fn [_conv-id _prof-id msgs]
+ (reset! messages msgs))
+ sr/async-cancellation-poll-interval-ms 5]
+ (testing "Closing body stream drops connection"
+ (let [body (mt/user-real-request :rasta :post 202 "ee/metabot-v3/agent-streaming"
+ {:request-options {:as :stream
+ :decompress-body false}}
+ {:message "Test closure"
+ :context {}
+ :conversation_id (str (random-uuid))
+ :history []
+ :state {}})]
+ (.read ^java.io.InputStream body) ;; start the handler
+ (.close ^java.io.Closeable body)
+ (u/poll {:thunk #(deref canceled)
+ :done? some?
+ :interval-ms 5})
+ (is (number? (:msg (first @messages)))
+ "store-messages! was called in the end on the lines streaming-request managed to process")
+ ;; if this flakes in CI, increase the number a bit; but it was 1 quite consistently for me
+ (is (> 10 (count @messages))
+ "But we shouldn't go through all 30 of them")
+ (testing "request to ai-service was canceled"
+ (is (< 20 @cnt) "Stopped writing when channel closed")
+ ;; see `metabase.server.streaming-response-test/canceling-chan-is-working-test` for explanation,
+ ;; reducing flakyness here
+ (is (some? @canceled)))))))))
+ (finally
+ (.stop ai-server)))))
(deftest feedback-endpoint-test
(mt/with-premium-features #{:metabot-v3}
diff --git a/enterprise/backend/test/metabase_enterprise/metabot_v3/client_test.clj b/enterprise/backend/test/metabase_enterprise/metabot_v3/client_test.clj
index 096fde38a89..d8d984dec38 100644
--- a/enterprise/backend/test/metabase_enterprise/metabot_v3/client_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/metabot_v3/client_test.clj
@@ -16,7 +16,7 @@
[metabase.util.malli :as mu]
[toucan2.core :as t2])
(:import
- (java.io ByteArrayOutputStream PipedInputStream PipedOutputStream)
+ (java.io ByteArrayOutputStream Closeable PipedInputStream PipedOutputStream)
(metabase.server.streaming_response StreamingResponse)))
(set! *warn-on-reflection* true)
@@ -41,9 +41,12 @@
(Thread/sleep ^long delay-ms))
(finally
(.close pipe))))
- {:status 200
- :headers {"content-type" "text/event-stream"}
- :body ret})))
+ {:status 200
+ :headers {"content-type" "text/event-stream"}
+ :body ret
+ :http-client (reify Closeable
+ (close [_this]
+ (.close ret)))})))
(defn consume-streaming-response
"Execute a StreamingResponse and capture its output"
diff --git a/test/metabase/server/streaming_response_test.clj b/test/metabase/server/streaming_response_test.clj
index b211a507248..21e6f89beb6 100644
--- a/test/metabase/server/streaming_response_test.clj
+++ b/test/metabase/server/streaming_response_test.clj
@@ -17,6 +17,7 @@
(:import
(jakarta.servlet AsyncContext ServletOutputStream)
(jakarta.servlet.http HttpServletResponse)
+ (java.io Closeable InputStream)
(java.util.concurrent Executors)
(org.apache.commons.lang3.concurrent BasicThreadFactory$Builder)))
@@ -157,7 +158,8 @@
handler (fn [req respond _raise]
(respond
(compojure.response/render
- (streaming-response/streaming-response {:content-type "text/event-stream; charset=utf-8"} [os canceled-chan]
+ (streaming-response/streaming-response {:content-type "text/event-stream; charset=utf-8"
+ :headers {"Transfer-Encoding" "chunked"}} [os canceled-chan]
(try
(loop []
(if (a/poll! canceled-chan)
@@ -182,15 +184,18 @@
(let [res (http/request {:method :post :url url
:as :stream
:decompress-body false})]
- (.read ^java.io.InputStream (:body res)) ;; start the handler
- (.close ^java.io.Closeable (:body res))
+ (.read ^InputStream (:body res)) ;; start the handler
+ ;; NOTE: this is the gist here, calling .close on the body will consume request *completely*
+ (.close ^Closeable (:http-client res))
(u/poll {:thunk #(deref canceled)
:done? some?
:interval-ms 5})
;; it's been 29 when I tested this, if it every becomes flaky maybe decrease the number?
(is (< 20 @cnt) "Stopped writing when channel closed")
- (testing "canceled-chan is working"
- (is (= :nice @canceled))))))
+ (testing "cancellation is working"
+ ;; we're not checking for particular way of cancelling, because cancellation poll interval can conflict
+ ;; with Thread/sleep and will make this test flaky
+ (is (some? @canceled))))))
(finally
(.stop server)))))