The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
metabot request cancelation (#65538)
Oleksandr Solovyovf91fa374af44
diff --git a/enterprise/backend/src/metabase_enterprise/metabot_v3/api.clj b/enterprise/backend/src/metabase_enterprise/metabot_v3/api.cljindex 68361385fc1..7c018fda754 100644--- a/enterprise/backend/src/metabase_enterprise/metabot_v3/api.clj+++ b/enterprise/backend/src/metabase_enterprise/metabot_v3/api.clj@@ -68,7 +68,7 @@:history history:state state:on-complete (fn [lines]- (store-message! conversation_id profile-id (metabot-v3.u/aisdk->messages "assistant" lines))+ (store-message! conversation_id profile-id (metabot-v3.u/aisdk->messages :assistant lines)):store-in-db)})))(api.macros/defendpoint :post "/agent-streaming"diff --git a/enterprise/backend/src/metabase_enterprise/metabot_v3/client.clj b/enterprise/backend/src/metabase_enterprise/metabot_v3/client.cljindex a9e92074900..9951850328a 100644--- a/enterprise/backend/src/metabase_enterprise/metabot_v3/client.clj+++ b/enterprise/backend/src/metabase_enterprise/metabot_v3/client.clj@@ -3,6 +3,7 @@[buddy.core.hash :as buddy-hash][buddy.sign.jwt :as jwt][clj-http.client :as http]+ [clojure.core.async :as a][clojure.java.io :as io][clojure.set :as set][clojure.string :as str]@@ -24,6 +25,7 @@[metabase.util.malli.registry :as mr][metabase.util.o11y :refer [with-span]])(:import+ (java.io BufferedReader)(org.eclipse.jetty.io EofException)))(set! *warn-on-reflection* true)@@ -154,7 +156,10 @@*debug* (assoc :debug true))response (post! url options)lines (when on-complete- (atom []))]+ (atom []))+ ;; NOTE: this atom is unused right now, but we potentially might put its value in database (see+ ;; `on-complete`) to indicate which requests were canceled in flight+ canceled (atom nil)](metabot-v3.context/log (:body response) :llm.log/llm->be)(log/debugf "Response from AI Proxy:\n%s" (u/pprint-to-str (select-keys response #{:body :status :headers})))(when-not (= (:status response) 200)@@ -163,21 +168,26 @@:response response})))(sr/streaming-response {:content-type "text/event-stream; charset=utf-8"} [os canceled-chan];; exiting with-open will close underlying request- (with-open [response-reader (io/reader (:body response))]- (try- ;; Response from the AI Service will send response parts separated by newline- (doseq [^String line (line-seq response-reader)]- (when on-complete- (swap! lines conj line))- (doto os- ;; `line-seq` strips newlines, so we need to append it back.- (.write (.getBytes line "UTF-8"))- (.write (.getBytes "\n"))- ;; Immediately flush so it feels fluid on the frontend- (.flush)))- ;; request was interrupted- (catch EofException _ nil)- (catch InterruptedException _ nil)))+ (with-open [^BufferedReader response-reader (io/reader (:body response))]+ ;; Response from the AI Service will send response parts separated by newline+ (loop [^String line (.readLine response-reader)]+ (cond+ (nil? line) nil+ (a/poll! canceled-chan) (do (reset! canceled true)+ (log/debug "Request cancelled"))+ :else+ (do+ (when on-complete+ (swap! lines conj line))+ (try+ (doto os+ ;; `line-seq` strips newlines, so we need to append it back.+ (.write (.getBytes line "UTF-8"))+ (.write (.getBytes "\n"))+ ;; Immediately flush so it feels fluid on the frontend+ (.flush))+ (catch EofException _ nil))+ (recur (.readLine response-reader))))))(when on-complete(on-complete @lines))))(catch Throwable ediff --git a/enterprise/backend/test/metabase_enterprise/metabot_v3/api_test.clj b/enterprise/backend/test/metabase_enterprise/metabot_v3/api_test.cljindex e0a9afad7e8..9cdf9bf2233 100644--- a/enterprise/backend/test/metabase_enterprise/metabot_v3/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/metabot_v3/api_test.clj@@ -9,7 +9,9 @@[metabase-enterprise.metabot-v3.client-test :as client-test][metabase-enterprise.metabot-v3.util :as metabot.u][metabase.search.test-util :as search.tu]+ [metabase.server.streaming-response :as sr][metabase.test :as mt]+ [metabase.util :as u][metabase.util.json :as json][toucan2.core :as t2]))@@ -66,28 +68,31 @@(let [mock-response (client-test/make-mock-stream-response(mapv #(str "msg-" % "\n") (range 30)){"some-model" {:prompt 12 :completion 3}})- messages (atom [])]+ messages (atom nil)](mt/test-helpers-set-global-values!(search.tu/with-index-disabled(mt/with-premium-features #{:metabot-v3}(with-redefs [client/post! (client-test/mock-post! mock-response {:delay-ms 5})api/store-message! (fn [_conv-id _prof-id msgs]- (reset! messages msgs))]+ (reset! messages msgs))+ sr/async-cancellation-poll-interval-ms 5](testing "Closing body stream drops connection"- (let [body (mt/user-real-request :rasta :post 202 "ee/metabot-v3/agent-streaming"- {:request-options {:as :stream- :decompress-body false}}- {:message "Test closure"- :context {}- :conversation_id (str (random-uuid))- :history []- :state {}})]- (.close body) ;; don't even need to read, it'll push first 3 lines immediately- (Thread/sleep 20) ;; no idea if this is enough to not flake, but let's see- (is (= "assistant" (:role (first @messages)))+ (let [body (mt/user-real-request :rasta :post 202 "ee/metabot-v3/agent-streaming"+ {:request-options {:as :stream+ :decompress-body false}}+ {:message "Test closure"+ :context {}+ :conversation_id (str (random-uuid))+ :history []+ :state {}})]+ (.read ^java.io.InputStream body) ;; start the handler+ (.close ^java.io.Closeable body)+ (u/poll {:thunk #(= :assistant (:role (first @messages)))+ :done? true?+ :interval-ms 5})+ (is (= :assistant (:role (first @messages)))"store-messages! was called in the end on the lines streaming-request managed to write to OS")- ;; it practically is 3 here all the time, and never 3; maybe it's `:output-buffer-size`?- ;; if this flakes in CI, increase the number a bit+ ;; if this flakes in CI, increase the number a bit; but it was 2 quite consistently for me(is (> 10 (-> @messages first :content str/split-lines count))"But we shouldn't go through all 30 of them")))))))))diff --git a/enterprise/backend/test/metabase_enterprise/metabot_v3/client_test.clj b/enterprise/backend/test/metabase_enterprise/metabot_v3/client_test.cljindex 50e87cf16f5..096fde38a89 100644--- a/enterprise/backend/test/metabase_enterprise/metabot_v3/client_test.clj+++ b/enterprise/backend/test/metabase_enterprise/metabot_v3/client_test.clj@@ -75,7 +75,7 @@(is (string? body))(is (=? [{:_type :TEXT :content "a1a2a3"}{:_type :FINISH_MESSAGE :finish_reason "stop"}]- (metabot-v3.util/aisdk->messages "assistant" (str/split-lines body))))))))))+ (metabot-v3.util/aisdk->messages :assistant (str/split-lines body))))))))))(deftest example-generation-payload-unknown-field-types-test(let [mp (mt/metadata-provider)]diff --git a/src/metabase/server/streaming_response.clj b/src/metabase/server/streaming_response.cljindex 7ca5293af91..2b1ec2982a7 100644--- a/src/metabase/server/streaming_response.clj+++ b/src/metabase/server/streaming_response.clj@@ -19,6 +19,7 @@(jakarta.servlet AsyncContext)(jakarta.servlet.http HttpServletResponse)(java.io BufferedWriter OutputStream OutputStreamWriter)+ (java.net SocketException)(java.nio ByteBuffer)(java.nio.channels ClosedChannelException SocketChannel)(java.nio.charset StandardCharsets)@@ -188,6 +189,7 @@false)))(catch InterruptedException _ false)(catch ClosedChannelException _ true)+ (catch SocketException _ true)(catch Throwable e(log/error e "Error determining whether HTTP request was canceled")false)))diff --git a/test/metabase/server/streaming_response_test.clj b/test/metabase/server/streaming_response_test.cljindex 88bcfbd1fdf..b211a507248 100644--- a/test/metabase/server/streaming_response_test.clj+++ b/test/metabase/server/streaming_response_test.clj@@ -151,7 +151,7 @@(Thread/sleep (long wait))(recur more))))))))))))-(deftest canceling-chan-is-not-working-test+(deftest canceling-chan-is-working-test(let [cnt (atom 30)canceled (atom nil)handler (fn [req respond _raise]@@ -160,18 +160,16 @@(streaming-response/streaming-response {:content-type "text/event-stream; charset=utf-8"} [os canceled-chan](try(loop []- (.write os (.getBytes (str "msg-" @cnt)))- (.write os (.getBytes "\n"))- (.flush os)- (swap! cnt dec)- (Thread/sleep 1)- ;; NOTE: we never get anything in there because `do-f*`, which puts stuff on canceled-chan,- ;; reacts to an exception which you get when you try to write to a closed OutputStream. So- ;; the body is interrupted.- (when (a/poll! canceled-chan)- (reset! canceled :nice))- (when (or (pos? @cnt) (nil? @canceled))- (recur)))+ (if (a/poll! canceled-chan)+ (reset! canceled :nice)+ (do+ (.write os (.getBytes (str "msg-" @cnt)))+ (.write os (.getBytes "\n"))+ (.flush os)+ (swap! cnt dec)+ (Thread/sleep 10)+ (when (pos? @cnt)+ (recur)))))(catch Exception _e(reset! canceled :not-nice))))req)))@@ -179,17 +177,20 @@.start)url (str "http://localhost:" (.. server getURI getPort))](try- (let [res (http/request {:method :post :url url- :as :stream- :decompress-body false})]- (.close ^java.io.Closeable (:body res)) ;; cancel the request- (Thread/sleep 10)- ;; it's been 28 when I tested this, if it every becomes flaky maybe decrease the number?- (is (< 20 @cnt) "Stopped writing when channel closed")- #_(testing "canceled-chan is working"- (is (= :nice @canceled) "Request has been canceled by looking at `canceled-chan`"))- (testing "canceled-chan is not working"- (is (= :not-nice @canceled) "Request has been canceled, but at what cost?")))+ (with-redefs [streaming-response/async-cancellation-poll-interval-ms 5]+ (testing "Closing body stops request handler"+ (let [res (http/request {:method :post :url url+ :as :stream+ :decompress-body false})]+ (.read ^java.io.InputStream (:body res)) ;; start the handler+ (.close ^java.io.Closeable (:body res))+ (u/poll {:thunk #(deref canceled)+ :done? some?+ :interval-ms 5})+ ;; it's been 29 when I tested this, if it every becomes flaky maybe decrease the number?+ (is (< 20 @cnt) "Stopped writing when channel closed")+ (testing "canceled-chan is working"+ (is (= :nice @canceled))))))(finally(.stop server)))))