Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

metabot request cancelation (#65538)

Oleksandr Solovyovf91fa374af44
diff --git a/enterprise/backend/src/metabase_enterprise/metabot_v3/api.clj b/enterprise/backend/src/metabase_enterprise/metabot_v3/api.clj
index 68361385fc1..7c018fda754 100644
--- a/enterprise/backend/src/metabase_enterprise/metabot_v3/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/metabot_v3/api.clj
@@ -68,7 +68,7 @@
:history history
:state state
:on-complete (fn [lines]
- (store-message! conversation_id profile-id (metabot-v3.u/aisdk->messages "assistant" lines))
+ (store-message! conversation_id profile-id (metabot-v3.u/aisdk->messages :assistant lines))
:store-in-db)})))
(api.macros/defendpoint :post "/agent-streaming"
diff --git a/enterprise/backend/src/metabase_enterprise/metabot_v3/client.clj b/enterprise/backend/src/metabase_enterprise/metabot_v3/client.clj
index a9e92074900..9951850328a 100644
--- a/enterprise/backend/src/metabase_enterprise/metabot_v3/client.clj
+++ b/enterprise/backend/src/metabase_enterprise/metabot_v3/client.clj
@@ -3,6 +3,7 @@
[buddy.core.hash :as buddy-hash]
[buddy.sign.jwt :as jwt]
[clj-http.client :as http]
+ [clojure.core.async :as a]
[clojure.java.io :as io]
[clojure.set :as set]
[clojure.string :as str]
@@ -24,6 +25,7 @@
[metabase.util.malli.registry :as mr]
[metabase.util.o11y :refer [with-span]])
(:import
+ (java.io BufferedReader)
(org.eclipse.jetty.io EofException)))
(set! *warn-on-reflection* true)
@@ -154,7 +156,10 @@
*debug* (assoc :debug true))
response (post! url options)
lines (when on-complete
- (atom []))]
+ (atom []))
+ ;; NOTE: this atom is unused right now, but we potentially might put its value in database (see
+ ;; `on-complete`) to indicate which requests were canceled in flight
+ canceled (atom nil)]
(metabot-v3.context/log (:body response) :llm.log/llm->be)
(log/debugf "Response from AI Proxy:\n%s" (u/pprint-to-str (select-keys response #{:body :status :headers})))
(when-not (= (:status response) 200)
@@ -163,21 +168,26 @@
:response response})))
(sr/streaming-response {:content-type "text/event-stream; charset=utf-8"} [os canceled-chan]
;; exiting with-open will close underlying request
- (with-open [response-reader (io/reader (:body response))]
- (try
- ;; Response from the AI Service will send response parts separated by newline
- (doseq [^String line (line-seq response-reader)]
- (when on-complete
- (swap! lines conj line))
- (doto os
- ;; `line-seq` strips newlines, so we need to append it back.
- (.write (.getBytes line "UTF-8"))
- (.write (.getBytes "\n"))
- ;; Immediately flush so it feels fluid on the frontend
- (.flush)))
- ;; request was interrupted
- (catch EofException _ nil)
- (catch InterruptedException _ nil)))
+ (with-open [^BufferedReader response-reader (io/reader (:body response))]
+ ;; Response from the AI Service will send response parts separated by newline
+ (loop [^String line (.readLine response-reader)]
+ (cond
+ (nil? line) nil
+ (a/poll! canceled-chan) (do (reset! canceled true)
+ (log/debug "Request cancelled"))
+ :else
+ (do
+ (when on-complete
+ (swap! lines conj line))
+ (try
+ (doto os
+ ;; `line-seq` strips newlines, so we need to append it back.
+ (.write (.getBytes line "UTF-8"))
+ (.write (.getBytes "\n"))
+ ;; Immediately flush so it feels fluid on the frontend
+ (.flush))
+ (catch EofException _ nil))
+ (recur (.readLine response-reader))))))
(when on-complete
(on-complete @lines))))
(catch Throwable e
diff --git a/enterprise/backend/test/metabase_enterprise/metabot_v3/api_test.clj b/enterprise/backend/test/metabase_enterprise/metabot_v3/api_test.clj
index e0a9afad7e8..9cdf9bf2233 100644
--- a/enterprise/backend/test/metabase_enterprise/metabot_v3/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/metabot_v3/api_test.clj
@@ -9,7 +9,9 @@
[metabase-enterprise.metabot-v3.client-test :as client-test]
[metabase-enterprise.metabot-v3.util :as metabot.u]
[metabase.search.test-util :as search.tu]
+ [metabase.server.streaming-response :as sr]
[metabase.test :as mt]
+ [metabase.util :as u]
[metabase.util.json :as json]
[toucan2.core :as t2]))
@@ -66,28 +68,31 @@
(let [mock-response (client-test/make-mock-stream-response
(mapv #(str "msg-" % "\n") (range 30))
{"some-model" {:prompt 12 :completion 3}})
- messages (atom [])]
+ messages (atom nil)]
(mt/test-helpers-set-global-values!
(search.tu/with-index-disabled
(mt/with-premium-features #{:metabot-v3}
(with-redefs [client/post! (client-test/mock-post! mock-response {:delay-ms 5})
api/store-message! (fn [_conv-id _prof-id msgs]
- (reset! messages msgs))]
+ (reset! messages msgs))
+ sr/async-cancellation-poll-interval-ms 5]
(testing "Closing body stream drops connection"
- (let [body (mt/user-real-request :rasta :post 202 "ee/metabot-v3/agent-streaming"
- {:request-options {:as :stream
- :decompress-body false}}
- {:message "Test closure"
- :context {}
- :conversation_id (str (random-uuid))
- :history []
- :state {}})]
- (.close body) ;; don't even need to read, it'll push first 3 lines immediately
- (Thread/sleep 20) ;; no idea if this is enough to not flake, but let's see
- (is (= "assistant" (:role (first @messages)))
+ (let [body (mt/user-real-request :rasta :post 202 "ee/metabot-v3/agent-streaming"
+ {:request-options {:as :stream
+ :decompress-body false}}
+ {:message "Test closure"
+ :context {}
+ :conversation_id (str (random-uuid))
+ :history []
+ :state {}})]
+ (.read ^java.io.InputStream body) ;; start the handler
+ (.close ^java.io.Closeable body)
+ (u/poll {:thunk #(= :assistant (:role (first @messages)))
+ :done? true?
+ :interval-ms 5})
+ (is (= :assistant (:role (first @messages)))
"store-messages! was called in the end on the lines streaming-request managed to write to OS")
- ;; it practically is 3 here all the time, and never 3; maybe it's `:output-buffer-size`?
- ;; if this flakes in CI, increase the number a bit
+ ;; if this flakes in CI, increase the number a bit; but it was 2 quite consistently for me
(is (> 10 (-> @messages first :content str/split-lines count))
"But we shouldn't go through all 30 of them")))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/metabot_v3/client_test.clj b/enterprise/backend/test/metabase_enterprise/metabot_v3/client_test.clj
index 50e87cf16f5..096fde38a89 100644
--- a/enterprise/backend/test/metabase_enterprise/metabot_v3/client_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/metabot_v3/client_test.clj
@@ -75,7 +75,7 @@
(is (string? body))
(is (=? [{:_type :TEXT :content "a1a2a3"}
{:_type :FINISH_MESSAGE :finish_reason "stop"}]
- (metabot-v3.util/aisdk->messages "assistant" (str/split-lines body))))))))))
+ (metabot-v3.util/aisdk->messages :assistant (str/split-lines body))))))))))
(deftest example-generation-payload-unknown-field-types-test
(let [mp (mt/metadata-provider)]
diff --git a/src/metabase/server/streaming_response.clj b/src/metabase/server/streaming_response.clj
index 7ca5293af91..2b1ec2982a7 100644
--- a/src/metabase/server/streaming_response.clj
+++ b/src/metabase/server/streaming_response.clj
@@ -19,6 +19,7 @@
(jakarta.servlet AsyncContext)
(jakarta.servlet.http HttpServletResponse)
(java.io BufferedWriter OutputStream OutputStreamWriter)
+ (java.net SocketException)
(java.nio ByteBuffer)
(java.nio.channels ClosedChannelException SocketChannel)
(java.nio.charset StandardCharsets)
@@ -188,6 +189,7 @@
false)))
(catch InterruptedException _ false)
(catch ClosedChannelException _ true)
+ (catch SocketException _ true)
(catch Throwable e
(log/error e "Error determining whether HTTP request was canceled")
false)))
diff --git a/test/metabase/server/streaming_response_test.clj b/test/metabase/server/streaming_response_test.clj
index 88bcfbd1fdf..b211a507248 100644
--- a/test/metabase/server/streaming_response_test.clj
+++ b/test/metabase/server/streaming_response_test.clj
@@ -151,7 +151,7 @@
(Thread/sleep (long wait))
(recur more))))))))))))
-(deftest canceling-chan-is-not-working-test
+(deftest canceling-chan-is-working-test
(let [cnt (atom 30)
canceled (atom nil)
handler (fn [req respond _raise]
@@ -160,18 +160,16 @@
(streaming-response/streaming-response {:content-type "text/event-stream; charset=utf-8"} [os canceled-chan]
(try
(loop []
- (.write os (.getBytes (str "msg-" @cnt)))
- (.write os (.getBytes "\n"))
- (.flush os)
- (swap! cnt dec)
- (Thread/sleep 1)
- ;; NOTE: we never get anything in there because `do-f*`, which puts stuff on canceled-chan,
- ;; reacts to an exception which you get when you try to write to a closed OutputStream. So
- ;; the body is interrupted.
- (when (a/poll! canceled-chan)
- (reset! canceled :nice))
- (when (or (pos? @cnt) (nil? @canceled))
- (recur)))
+ (if (a/poll! canceled-chan)
+ (reset! canceled :nice)
+ (do
+ (.write os (.getBytes (str "msg-" @cnt)))
+ (.write os (.getBytes "\n"))
+ (.flush os)
+ (swap! cnt dec)
+ (Thread/sleep 10)
+ (when (pos? @cnt)
+ (recur)))))
(catch Exception _e
(reset! canceled :not-nice))))
req)))
@@ -179,17 +177,20 @@
.start)
url (str "http://localhost:" (.. server getURI getPort))]
(try
- (let [res (http/request {:method :post :url url
- :as :stream
- :decompress-body false})]
- (.close ^java.io.Closeable (:body res)) ;; cancel the request
- (Thread/sleep 10)
- ;; it's been 28 when I tested this, if it every becomes flaky maybe decrease the number?
- (is (< 20 @cnt) "Stopped writing when channel closed")
- #_(testing "canceled-chan is working"
- (is (= :nice @canceled) "Request has been canceled by looking at `canceled-chan`"))
- (testing "canceled-chan is not working"
- (is (= :not-nice @canceled) "Request has been canceled, but at what cost?")))
+ (with-redefs [streaming-response/async-cancellation-poll-interval-ms 5]
+ (testing "Closing body stops request handler"
+ (let [res (http/request {:method :post :url url
+ :as :stream
+ :decompress-body false})]
+ (.read ^java.io.InputStream (:body res)) ;; start the handler
+ (.close ^java.io.Closeable (:body res))
+ (u/poll {:thunk #(deref canceled)
+ :done? some?
+ :interval-ms 5})
+ ;; it's been 29 when I tested this, if it every becomes flaky maybe decrease the number?
+ (is (< 20 @cnt) "Stopped writing when channel closed")
+ (testing "canceled-chan is working"
+ (is (= :nice @canceled))))))
(finally
(.stop server)))))