Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

metabot: handle (and propagate) request cancellation (#65461)

Oleksandr Solovyov6ff47bfe2235
diff --git a/enterprise/backend/src/metabase_enterprise/metabot_v3/client.clj b/enterprise/backend/src/metabase_enterprise/metabot_v3/client.clj
index 81c8702071d..a9e92074900 100644
--- a/enterprise/backend/src/metabase_enterprise/metabot_v3/client.clj
+++ b/enterprise/backend/src/metabase_enterprise/metabot_v3/client.clj
@@ -22,7 +22,9 @@
[metabase.util.log :as log]
[metabase.util.malli :as mu]
[metabase.util.malli.registry :as mr]
- [metabase.util.o11y :refer [with-span]]))
+ [metabase.util.o11y :refer [with-span]])
+ (:import
+ (org.eclipse.jetty.io EofException)))
(set! *warn-on-reflection* true)
@@ -155,26 +157,29 @@
(atom []))]
(metabot-v3.context/log (:body response) :llm.log/llm->be)
(log/debugf "Response from AI Proxy:\n%s" (u/pprint-to-str (select-keys response #{:body :status :headers})))
- (if (= (:status response) 200)
- ;; TODO: handle canceled-chan here?
- (sr/streaming-response {:content-type "text/event-stream; charset=utf-8"} [os canceled-chan]
- ;; Response from the AI Service will send response parts separated by newline
- (with-open [response-reader (io/reader (:body response))]
+ (when-not (= (:status response) 200)
+ (throw (ex-info (format "Error: unexpected status code: %d %s" (:status response) (:reason-phrase response))
+ {:request (assoc options :body body)
+ :response response})))
+ (sr/streaming-response {:content-type "text/event-stream; charset=utf-8"} [os canceled-chan]
+ ;; exiting with-open will close underlying request
+ (with-open [response-reader (io/reader (:body response))]
+ (try
+ ;; Response from the AI Service will send response parts separated by newline
(doseq [^String line (line-seq response-reader)]
(when on-complete
(swap! lines conj line))
(doto os
- ;; Line we read from response will lack newline at the end, but FE will be confused without that, so
- ;; we need to append it.
+ ;; `line-seq` strips newlines, so we need to append it back.
(.write (.getBytes line "UTF-8"))
(.write (.getBytes "\n"))
;; Immediately flush so it feels fluid on the frontend
- (.flush))))
- (when on-complete
- (on-complete @lines)))
- (throw (ex-info (format "Error: unexpected status code: %d %s" (:status response) (:reason-phrase response))
- {:request (assoc options :body body)
- :response response}))))
+ (.flush)))
+ ;; request was interrupted
+ (catch EofException _ nil)
+ (catch InterruptedException _ nil)))
+ (when on-complete
+ (on-complete @lines))))
(catch Throwable e
(throw (ex-info (format "Error in request to AI Proxy: %s" (ex-message e)) {} e)))))
diff --git a/enterprise/backend/test/metabase_enterprise/metabot_v3/api_test.clj b/enterprise/backend/test/metabase_enterprise/metabot_v3/api_test.clj
index d1eb86acce8..e0a9afad7e8 100644
--- a/enterprise/backend/test/metabase_enterprise/metabot_v3/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/metabot_v3/api_test.clj
@@ -4,8 +4,11 @@
[clojure.string :as str]
[clojure.test :refer :all]
[medley.core :as m]
+ [metabase-enterprise.metabot-v3.api :as api]
+ [metabase-enterprise.metabot-v3.client :as client]
[metabase-enterprise.metabot-v3.client-test :as client-test]
[metabase-enterprise.metabot-v3.util :as metabot.u]
+ [metabase.search.test-util :as search.tu]
[metabase.test :as mt]
[metabase.util.json :as json]
[toucan2.core :as t2]))
@@ -21,10 +24,10 @@
question {:role "user" :content "Test streaming question"}
historical-message {:role "user" :content "previous message"}
ai-requests (atom [])]
- (mt/with-dynamic-fn-redefs [http/post (fn [url opts]
- (swap! ai-requests conj (-> (String. ^bytes (:body opts) "UTF-8")
- json/decode+kw))
- ((client-test/mock-post! mock-response) url opts))]
+ (mt/with-dynamic-fn-redefs [client/post! (fn [url opts]
+ (swap! ai-requests conj (-> (String. ^bytes (:body opts) "UTF-8")
+ json/decode+kw))
+ ((client-test/mock-post! mock-response) url opts))]
(testing "Streaming request"
(doseq [metabot-id [nil (str (random-uuid))]]
(mt/with-model-cleanup [:model/MetabotMessage
@@ -59,6 +62,35 @@
:data [{:role "assistant" :content "Hello from streaming!"}]}]
messages))))))))))
+(deftest closing-connection-test
+ (let [mock-response (client-test/make-mock-stream-response
+ (mapv #(str "msg-" % "\n") (range 30))
+ {"some-model" {:prompt 12 :completion 3}})
+ messages (atom [])]
+ (mt/test-helpers-set-global-values!
+ (search.tu/with-index-disabled
+ (mt/with-premium-features #{:metabot-v3}
+ (with-redefs [client/post! (client-test/mock-post! mock-response {:delay-ms 5})
+ api/store-message! (fn [_conv-id _prof-id msgs]
+ (reset! messages msgs))]
+ (testing "Closing body stream drops connection"
+ (let [body (mt/user-real-request :rasta :post 202 "ee/metabot-v3/agent-streaming"
+ {:request-options {:as :stream
+ :decompress-body false}}
+ {:message "Test closure"
+ :context {}
+ :conversation_id (str (random-uuid))
+ :history []
+ :state {}})]
+ (.close body) ;; don't even need to read, it'll push first 3 lines immediately
+ (Thread/sleep 20) ;; no idea if this is enough to not flake, but let's see
+ (is (= "assistant" (:role (first @messages)))
+ "store-messages! was called in the end on the lines streaming-request managed to write to OS")
+ ;; it practically is 3 here all the time, and never 3; maybe it's `:output-buffer-size`?
+ ;; if this flakes in CI, increase the number a bit
+ (is (> 10 (-> @messages first :content str/split-lines count))
+ "But we shouldn't go through all 30 of them")))))))))
+
(deftest feedback-endpoint-test
(mt/with-premium-features #{:metabot-v3}
(mt/with-random-premium-token! [premium-token]
diff --git a/enterprise/backend/test/metabase_enterprise/metabot_v3/client_test.clj b/enterprise/backend/test/metabase_enterprise/metabot_v3/client_test.clj
index 05be8b749d7..50e87cf16f5 100644
--- a/enterprise/backend/test/metabase_enterprise/metabot_v3/client_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/metabot_v3/client_test.clj
@@ -16,7 +16,7 @@
[metabase.util.malli :as mu]
[toucan2.core :as t2])
(:import
- (java.io ByteArrayInputStream ByteArrayOutputStream)
+ (java.io ByteArrayOutputStream PipedInputStream PipedOutputStream)
(metabase.server.streaming_response StreamingResponse)))
(set! *warn-on-reflection* true)
@@ -27,11 +27,23 @@
(str/join))
"d:" (json/encode {:finishReason "stop" :usage usage})))
-(defn mock-post! [^String body]
+(defn mock-post! [^String body & [{:keys [delay-ms]
+ :or {delay-ms 0}}]]
(fn [_url _opts]
- {:status 200
- :headers {"content-type" "text/event-stream"}
- :body (ByteArrayInputStream. (.getBytes body "UTF-8"))}))
+ (let [ret (PipedInputStream.)
+ pipe (PipedOutputStream. ret)]
+ (future
+ (try
+ (doseq [^String line (str/split body #"\n")]
+ (.write pipe (.getBytes line "UTF-8"))
+ (.write pipe (.getBytes "\n" "UTF-8"))
+ (.flush pipe)
+ (Thread/sleep ^long delay-ms))
+ (finally
+ (.close pipe))))
+ {:status 200
+ :headers {"content-type" "text/event-stream"}
+ :body ret})))
(defn consume-streaming-response
"Execute a StreamingResponse and capture its output"
diff --git a/src/metabase/server/streaming_response.clj b/src/metabase/server/streaming_response.clj
index 9309bf5769e..7ca5293af91 100644
--- a/src/metabase/server/streaming_response.clj
+++ b/src/metabase/server/streaming_response.clj
@@ -307,6 +307,7 @@
(write-something-to-stream! os))
`f` should block until it is completely finished writing to the stream, which will be closed thereafter.
+ NOTE: `canceled-chan` **IS NOT WORKING**; see `metabase.server.streaming-response-test/canceling-response-2`
`canceled-chan` can be monitored to see if the request is canceled before results are fully written to the stream.
Current options:
diff --git a/test/metabase/server/streaming_response_test.clj b/test/metabase/server/streaming_response_test.clj
index c136014ac1b..88bcfbd1fdf 100644
--- a/test/metabase/server/streaming_response_test.clj
+++ b/test/metabase/server/streaming_response_test.clj
@@ -3,8 +3,10 @@
[clj-http.client :as http]
[clojure.core.async :as a]
[clojure.test :refer :all]
+ [compojure.response]
[metabase.driver :as driver]
[metabase.query-processor.pipeline :as qp.pipeline]
+ [metabase.server.instance :as server.instance]
[metabase.server.protocols :as server.protocols]
[metabase.server.streaming-response :as streaming-response]
[metabase.server.streaming-response.thread-pool :as thread-pool]
@@ -139,16 +141,58 @@
nil)
futur (http/post url (assoc request :async? true) identity (fn [e] (throw e)))]
(is (future? futur))
- ;; wait a little while for the query to start running -- this should usually happen fairly quickly
+ ;; wait a little while for the query to start running -- this should usually happen fairly quickly
(mt/wait-for-result start-chan (u/seconds->ms 15))
(future-cancel futur)
- ;; check every 10ms, up to 1000ms, whether `canceled?` is now `true`
+ ;; check every 10ms, up to 1000ms, whether `canceled?` is now `true`
(is (loop [[wait & more] (repeat 10 100)]
(or @canceled?
(when wait
(Thread/sleep (long wait))
(recur more))))))))))))
+(deftest canceling-chan-is-not-working-test
+ (let [cnt (atom 30)
+ canceled (atom nil)
+ handler (fn [req respond _raise]
+ (respond
+ (compojure.response/render
+ (streaming-response/streaming-response {:content-type "text/event-stream; charset=utf-8"} [os canceled-chan]
+ (try
+ (loop []
+ (.write os (.getBytes (str "msg-" @cnt)))
+ (.write os (.getBytes "\n"))
+ (.flush os)
+ (swap! cnt dec)
+ (Thread/sleep 1)
+ ;; NOTE: we never get anything in there because `do-f*`, which puts stuff on canceled-chan,
+ ;; reacts to an exception which you get when you try to write to a closed OutputStream. So
+ ;; the body is interrupted.
+ (when (a/poll! canceled-chan)
+ (reset! canceled :nice))
+ (when (or (pos? @cnt) (nil? @canceled))
+ (recur)))
+ (catch Exception _e
+ (reset! canceled :not-nice))))
+ req)))
+ server (doto (server.instance/create-server handler {:port 0 :join? false})
+ .start)
+ url (str "http://localhost:" (.. server getURI getPort))]
+ (try
+ (let [res (http/request {:method :post :url url
+ :as :stream
+ :decompress-body false})]
+ (.close ^java.io.Closeable (:body res)) ;; cancel the request
+ (Thread/sleep 10)
+ ;; it's been 28 when I tested this, if it every becomes flaky maybe decrease the number?
+ (is (< 20 @cnt) "Stopped writing when channel closed")
+ #_(testing "canceled-chan is working"
+ (is (= :nice @canceled) "Request has been canceled by looking at `canceled-chan`"))
+ (testing "canceled-chan is not working"
+ (is (= :not-nice @canceled) "Request has been canceled, but at what cost?")))
+ (finally
+ (.stop server)))))
+
(def ^:private ^:dynamic *number-of-cans* nil)
(deftest ^:parallel preserve-bindings-test