The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
metabot: handle (and propagate) request cancellation (#65461)
Oleksandr Solovyov6ff47bfe2235
diff --git a/enterprise/backend/src/metabase_enterprise/metabot_v3/client.clj b/enterprise/backend/src/metabase_enterprise/metabot_v3/client.cljindex 81c8702071d..a9e92074900 100644--- a/enterprise/backend/src/metabase_enterprise/metabot_v3/client.clj+++ b/enterprise/backend/src/metabase_enterprise/metabot_v3/client.clj@@ -22,7 +22,9 @@[metabase.util.log :as log][metabase.util.malli :as mu][metabase.util.malli.registry :as mr]- [metabase.util.o11y :refer [with-span]]))+ [metabase.util.o11y :refer [with-span]])+ (:import+ (org.eclipse.jetty.io EofException)))(set! *warn-on-reflection* true)@@ -155,26 +157,29 @@(atom []))](metabot-v3.context/log (:body response) :llm.log/llm->be)(log/debugf "Response from AI Proxy:\n%s" (u/pprint-to-str (select-keys response #{:body :status :headers})))- (if (= (:status response) 200)- ;; TODO: handle canceled-chan here?- (sr/streaming-response {:content-type "text/event-stream; charset=utf-8"} [os canceled-chan]- ;; Response from the AI Service will send response parts separated by newline- (with-open [response-reader (io/reader (:body response))]+ (when-not (= (:status response) 200)+ (throw (ex-info (format "Error: unexpected status code: %d %s" (:status response) (:reason-phrase response))+ {:request (assoc options :body body)+ :response response})))+ (sr/streaming-response {:content-type "text/event-stream; charset=utf-8"} [os canceled-chan]+ ;; exiting with-open will close underlying request+ (with-open [response-reader (io/reader (:body response))]+ (try+ ;; Response from the AI Service will send response parts separated by newline(doseq [^String line (line-seq response-reader)](when on-complete(swap! lines conj line))(doto os- ;; Line we read from response will lack newline at the end, but FE will be confused without that, so- ;; we need to append it.+ ;; `line-seq` strips newlines, so we need to append it back.(.write (.getBytes line "UTF-8"))(.write (.getBytes "\n"));; Immediately flush so it feels fluid on the frontend- (.flush))))- (when on-complete- (on-complete @lines)))- (throw (ex-info (format "Error: unexpected status code: %d %s" (:status response) (:reason-phrase response))- {:request (assoc options :body body)- :response response}))))+ (.flush)))+ ;; request was interrupted+ (catch EofException _ nil)+ (catch InterruptedException _ nil)))+ (when on-complete+ (on-complete @lines))))(catch Throwable e(throw (ex-info (format "Error in request to AI Proxy: %s" (ex-message e)) {} e)))))diff --git a/enterprise/backend/test/metabase_enterprise/metabot_v3/api_test.clj b/enterprise/backend/test/metabase_enterprise/metabot_v3/api_test.cljindex d1eb86acce8..e0a9afad7e8 100644--- a/enterprise/backend/test/metabase_enterprise/metabot_v3/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/metabot_v3/api_test.clj@@ -4,8 +4,11 @@[clojure.string :as str][clojure.test :refer :all][medley.core :as m]+ [metabase-enterprise.metabot-v3.api :as api]+ [metabase-enterprise.metabot-v3.client :as client][metabase-enterprise.metabot-v3.client-test :as client-test][metabase-enterprise.metabot-v3.util :as metabot.u]+ [metabase.search.test-util :as search.tu][metabase.test :as mt][metabase.util.json :as json][toucan2.core :as t2]))@@ -21,10 +24,10 @@question {:role "user" :content "Test streaming question"}historical-message {:role "user" :content "previous message"}ai-requests (atom [])]- (mt/with-dynamic-fn-redefs [http/post (fn [url opts]- (swap! ai-requests conj (-> (String. ^bytes (:body opts) "UTF-8")- json/decode+kw))- ((client-test/mock-post! mock-response) url opts))]+ (mt/with-dynamic-fn-redefs [client/post! (fn [url opts]+ (swap! ai-requests conj (-> (String. ^bytes (:body opts) "UTF-8")+ json/decode+kw))+ ((client-test/mock-post! mock-response) url opts))](testing "Streaming request"(doseq [metabot-id [nil (str (random-uuid))]](mt/with-model-cleanup [:model/MetabotMessage@@ -59,6 +62,35 @@:data [{:role "assistant" :content "Hello from streaming!"}]}]messages))))))))))+(deftest closing-connection-test+ (let [mock-response (client-test/make-mock-stream-response+ (mapv #(str "msg-" % "\n") (range 30))+ {"some-model" {:prompt 12 :completion 3}})+ messages (atom [])]+ (mt/test-helpers-set-global-values!+ (search.tu/with-index-disabled+ (mt/with-premium-features #{:metabot-v3}+ (with-redefs [client/post! (client-test/mock-post! mock-response {:delay-ms 5})+ api/store-message! (fn [_conv-id _prof-id msgs]+ (reset! messages msgs))]+ (testing "Closing body stream drops connection"+ (let [body (mt/user-real-request :rasta :post 202 "ee/metabot-v3/agent-streaming"+ {:request-options {:as :stream+ :decompress-body false}}+ {:message "Test closure"+ :context {}+ :conversation_id (str (random-uuid))+ :history []+ :state {}})]+ (.close body) ;; don't even need to read, it'll push first 3 lines immediately+ (Thread/sleep 20) ;; no idea if this is enough to not flake, but let's see+ (is (= "assistant" (:role (first @messages)))+ "store-messages! was called in the end on the lines streaming-request managed to write to OS")+ ;; it practically is 3 here all the time, and never 3; maybe it's `:output-buffer-size`?+ ;; if this flakes in CI, increase the number a bit+ (is (> 10 (-> @messages first :content str/split-lines count))+ "But we shouldn't go through all 30 of them")))))))))+(deftest feedback-endpoint-test(mt/with-premium-features #{:metabot-v3}(mt/with-random-premium-token! [premium-token]diff --git a/enterprise/backend/test/metabase_enterprise/metabot_v3/client_test.clj b/enterprise/backend/test/metabase_enterprise/metabot_v3/client_test.cljindex 05be8b749d7..50e87cf16f5 100644--- a/enterprise/backend/test/metabase_enterprise/metabot_v3/client_test.clj+++ b/enterprise/backend/test/metabase_enterprise/metabot_v3/client_test.clj@@ -16,7 +16,7 @@[metabase.util.malli :as mu][toucan2.core :as t2])(:import- (java.io ByteArrayInputStream ByteArrayOutputStream)+ (java.io ByteArrayOutputStream PipedInputStream PipedOutputStream)(metabase.server.streaming_response StreamingResponse)))(set! *warn-on-reflection* true)@@ -27,11 +27,23 @@(str/join))"d:" (json/encode {:finishReason "stop" :usage usage})))-(defn mock-post! [^String body]+(defn mock-post! [^String body & [{:keys [delay-ms]+ :or {delay-ms 0}}]](fn [_url _opts]- {:status 200- :headers {"content-type" "text/event-stream"}- :body (ByteArrayInputStream. (.getBytes body "UTF-8"))}))+ (let [ret (PipedInputStream.)+ pipe (PipedOutputStream. ret)]+ (future+ (try+ (doseq [^String line (str/split body #"\n")]+ (.write pipe (.getBytes line "UTF-8"))+ (.write pipe (.getBytes "\n" "UTF-8"))+ (.flush pipe)+ (Thread/sleep ^long delay-ms))+ (finally+ (.close pipe))))+ {:status 200+ :headers {"content-type" "text/event-stream"}+ :body ret})))(defn consume-streaming-response"Execute a StreamingResponse and capture its output"diff --git a/src/metabase/server/streaming_response.clj b/src/metabase/server/streaming_response.cljindex 9309bf5769e..7ca5293af91 100644--- a/src/metabase/server/streaming_response.clj+++ b/src/metabase/server/streaming_response.clj@@ -307,6 +307,7 @@(write-something-to-stream! os))`f` should block until it is completely finished writing to the stream, which will be closed thereafter.+ NOTE: `canceled-chan` **IS NOT WORKING**; see `metabase.server.streaming-response-test/canceling-response-2``canceled-chan` can be monitored to see if the request is canceled before results are fully written to the stream.Current options:diff --git a/test/metabase/server/streaming_response_test.clj b/test/metabase/server/streaming_response_test.cljindex c136014ac1b..88bcfbd1fdf 100644--- a/test/metabase/server/streaming_response_test.clj+++ b/test/metabase/server/streaming_response_test.clj@@ -3,8 +3,10 @@[clj-http.client :as http][clojure.core.async :as a][clojure.test :refer :all]+ [compojure.response][metabase.driver :as driver][metabase.query-processor.pipeline :as qp.pipeline]+ [metabase.server.instance :as server.instance][metabase.server.protocols :as server.protocols][metabase.server.streaming-response :as streaming-response][metabase.server.streaming-response.thread-pool :as thread-pool]@@ -139,16 +141,58 @@nil)futur (http/post url (assoc request :async? true) identity (fn [e] (throw e)))](is (future? futur))- ;; wait a little while for the query to start running -- this should usually happen fairly quickly+ ;; wait a little while for the query to start running -- this should usually happen fairly quickly(mt/wait-for-result start-chan (u/seconds->ms 15))(future-cancel futur)- ;; check every 10ms, up to 1000ms, whether `canceled?` is now `true`+ ;; check every 10ms, up to 1000ms, whether `canceled?` is now `true`(is (loop [[wait & more] (repeat 10 100)](or @canceled?(when wait(Thread/sleep (long wait))(recur more))))))))))))+(deftest canceling-chan-is-not-working-test+ (let [cnt (atom 30)+ canceled (atom nil)+ handler (fn [req respond _raise]+ (respond+ (compojure.response/render+ (streaming-response/streaming-response {:content-type "text/event-stream; charset=utf-8"} [os canceled-chan]+ (try+ (loop []+ (.write os (.getBytes (str "msg-" @cnt)))+ (.write os (.getBytes "\n"))+ (.flush os)+ (swap! cnt dec)+ (Thread/sleep 1)+ ;; NOTE: we never get anything in there because `do-f*`, which puts stuff on canceled-chan,+ ;; reacts to an exception which you get when you try to write to a closed OutputStream. So+ ;; the body is interrupted.+ (when (a/poll! canceled-chan)+ (reset! canceled :nice))+ (when (or (pos? @cnt) (nil? @canceled))+ (recur)))+ (catch Exception _e+ (reset! canceled :not-nice))))+ req)))+ server (doto (server.instance/create-server handler {:port 0 :join? false})+ .start)+ url (str "http://localhost:" (.. server getURI getPort))]+ (try+ (let [res (http/request {:method :post :url url+ :as :stream+ :decompress-body false})]+ (.close ^java.io.Closeable (:body res)) ;; cancel the request+ (Thread/sleep 10)+ ;; it's been 28 when I tested this, if it every becomes flaky maybe decrease the number?+ (is (< 20 @cnt) "Stopped writing when channel closed")+ #_(testing "canceled-chan is working"+ (is (= :nice @canceled) "Request has been canceled by looking at `canceled-chan`"))+ (testing "canceled-chan is not working"+ (is (= :not-nice @canceled) "Request has been canceled, but at what cost?")))+ (finally+ (.stop server)))))+(def ^:private ^:dynamic *number-of-cans* nil)(deftest ^:parallel preserve-bindings-test