The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
request cancelation: this time for real (#65660)
Oleksandr Solovyov559d3357a2a3
diff --git a/enterprise/backend/src/metabase_enterprise/metabot_v3/client.clj b/enterprise/backend/src/metabase_enterprise/metabot_v3/client.cljindex 9951850328a..44bf8ed77c8 100644--- a/enterprise/backend/src/metabase_enterprise/metabot_v3/client.clj+++ b/enterprise/backend/src/metabase_enterprise/metabot_v3/client.clj@@ -25,7 +25,7 @@[metabase.util.malli.registry :as mr][metabase.util.o11y :refer [with-span]])(:import- (java.io BufferedReader)+ (java.io BufferedReader Closeable FilterInputStream InputStream)(org.eclipse.jetty.io EofException)))(set! *warn-on-reflection* true)@@ -108,6 +108,19 @@(defn- generate-embeddings-endpoint [](str (metabot-v3.settings/ai-service-base-url) "/v1/embeddings"))+(defn- quick-closing-body+ "Some requests come with body wrapped in ContentLengthInputStream, and that will never close the underlying stream.+ So we just close the client itself.++ Please use `Connection: close` header when using this function to prevent connection being reused by HttpClient.++ See: https://github.com/dakrone/clj-http/issues/627+ Also: metabase-enterprise.metabot-v3.api-test/closing-connection-test (for 'chunked')"+ [response]+ (proxy [FilterInputStream] [^InputStream (:body response)]+ (close []+ (.close ^Closeable (:http-client response)))))+(mu/defn streaming-request :- :any"Make a streaming V2 request to the AI Service@@ -149,10 +162,16 @@"Content-Type" "application/json;charset=UTF-8""x-metabase-instance-token" (premium-features/premium-embedding-token)"x-metabase-session-token" session-id- "x-metabase-url" (system/site-url)}+ "x-metabase-url" (system/site-url)+ ;; close conn so it's not reused so that when we use+ ;; `quick-closing-body` there are no weird problems+ "Connection" "close"}:body (->json-bytes body):throw-exceptions false- :as :stream}+ :as :stream+ ;; Don't compress streaming responses - adds latency and breaks+ ;; cancellation when streams are incomplete+ :decompress-body false}*debug* (assoc :debug true))response (post! url options)lines (when on-complete@@ -162,13 +181,13 @@canceled (atom nil)](metabot-v3.context/log (:body response) :llm.log/llm->be)(log/debugf "Response from AI Proxy:\n%s" (u/pprint-to-str (select-keys response #{:body :status :headers})))- (when-not (= (:status response) 200)+ (when-not (#{200 202} (:status response))(throw (ex-info (format "Error: unexpected status code: %d %s" (:status response) (:reason-phrase response)){:request (assoc options :body body):response response})))(sr/streaming-response {:content-type "text/event-stream; charset=utf-8"} [os canceled-chan]- ;; exiting with-open will close underlying request- (with-open [^BufferedReader response-reader (io/reader (:body response))]+ ;; see `quick-closing-body` docs and see `Connection` header supporting this behavior+ (with-open [^BufferedReader response-reader (io/reader (quick-closing-body response))];; Response from the AI Service will send response parts separated by newline(loop [^String line (.readLine response-reader)](cond@@ -186,8 +205,12 @@(.write (.getBytes "\n"));; Immediately flush so it feels fluid on the frontend(.flush))- (catch EofException _ nil))- (recur (.readLine response-reader))))))+ (catch EofException _+ (reset! canceled true)+ (log/debug "Request cancelled, through exception")))+ (when-not @canceled+ ;; `recur` cannot be inside of `try`, so we have to signal stop somehow+ (recur (.readLine response-reader)))))))(when on-complete(on-complete @lines))))(catch Throwable ediff --git a/enterprise/backend/test/metabase_enterprise/metabot_v3/api_test.clj b/enterprise/backend/test/metabase_enterprise/metabot_v3/api_test.cljindex 9cdf9bf2233..22dff4094a4 100644--- a/enterprise/backend/test/metabase_enterprise/metabot_v3/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/metabot_v3/api_test.clj@@ -1,14 +1,17 @@(ns metabase-enterprise.metabot-v3.api-test(:require[clj-http.client :as http]+ [clojure.core.async :as a][clojure.string :as str][clojure.test :refer :all]+ [compojure.response][medley.core :as m][metabase-enterprise.metabot-v3.api :as api][metabase-enterprise.metabot-v3.client :as client][metabase-enterprise.metabot-v3.client-test :as client-test][metabase-enterprise.metabot-v3.util :as metabot.u][metabase.search.test-util :as search.tu]+ [metabase.server.instance :as server.instance][metabase.server.streaming-response :as sr][metabase.test :as mt][metabase.util :as u]@@ -65,36 +68,67 @@messages))))))))))(deftest closing-connection-test- (let [mock-response (client-test/make-mock-stream-response- (mapv #(str "msg-" % "\n") (range 30))- {"some-model" {:prompt 12 :completion 3}})- messages (atom nil)]- (mt/test-helpers-set-global-values!- (search.tu/with-index-disabled- (mt/with-premium-features #{:metabot-v3}- (with-redefs [client/post! (client-test/mock-post! mock-response {:delay-ms 5})- api/store-message! (fn [_conv-id _prof-id msgs]- (reset! messages msgs))- sr/async-cancellation-poll-interval-ms 5]- (testing "Closing body stream drops connection"- (let [body (mt/user-real-request :rasta :post 202 "ee/metabot-v3/agent-streaming"- {:request-options {:as :stream- :decompress-body false}}- {:message "Test closure"- :context {}- :conversation_id (str (random-uuid))- :history []- :state {}})]- (.read ^java.io.InputStream body) ;; start the handler- (.close ^java.io.Closeable body)- (u/poll {:thunk #(= :assistant (:role (first @messages)))- :done? true?- :interval-ms 5})- (is (= :assistant (:role (first @messages)))- "store-messages! was called in the end on the lines streaming-request managed to write to OS")- ;; if this flakes in CI, increase the number a bit; but it was 2 quite consistently for me- (is (> 10 (-> @messages first :content str/split-lines count))- "But we shouldn't go through all 30 of them")))))))))+ (let [messages (atom nil)+ cnt (atom 30)+ canceled (atom nil)+ ai-handler (fn [req respond _raise]+ (respond+ (compojure.response/render+ (sr/streaming-response {:content-type "text/event-stream; charset=utf-8"+ ;; Transfer-Encoding: chunked is what makes normal .close on body fail+ ;; See: `metabase-enterprise.metabot-v3.client/quick-closing-body`+ :headers {"Transfer-Encoding" "chunked"}} [os canceled-chan]+ (try+ (loop []+ (if (a/poll! canceled-chan)+ (reset! canceled :nice)+ (do+ (.write os (.getBytes (str "2:" (json/encode {:msg @cnt}) "\n")))+ (.flush os)+ (swap! cnt dec)+ (Thread/sleep 10)+ (when (pos? @cnt)+ (recur)))))+ (catch Exception _e+ (reset! canceled :not-nice))))+ req)))+ ai-server (doto (server.instance/create-server ai-handler {:port 0 :join? false})+ .start)+ ai-url (str "http://localhost:" (.. ai-server getURI getPort))]+ (try+ (mt/test-helpers-set-global-values!+ (search.tu/with-index-disabled+ (mt/with-premium-features #{:metabot-v3}+ (with-redefs [client/ai-url (constantly ai-url)+ api/store-message! (fn [_conv-id _prof-id msgs]+ (reset! messages msgs))+ sr/async-cancellation-poll-interval-ms 5]+ (testing "Closing body stream drops connection"+ (let [body (mt/user-real-request :rasta :post 202 "ee/metabot-v3/agent-streaming"+ {:request-options {:as :stream+ :decompress-body false}}+ {:message "Test closure"+ :context {}+ :conversation_id (str (random-uuid))+ :history []+ :state {}})]+ (.read ^java.io.InputStream body) ;; start the handler+ (.close ^java.io.Closeable body)+ (u/poll {:thunk #(deref canceled)+ :done? some?+ :interval-ms 5})+ (is (number? (:msg (first @messages)))+ "store-messages! was called in the end on the lines streaming-request managed to process")+ ;; if this flakes in CI, increase the number a bit; but it was 1 quite consistently for me+ (is (> 10 (count @messages))+ "But we shouldn't go through all 30 of them")+ (testing "request to ai-service was canceled"+ (is (< 20 @cnt) "Stopped writing when channel closed")+ ;; see `metabase.server.streaming-response-test/canceling-chan-is-working-test` for explanation,+ ;; reducing flakyness here+ (is (some? @canceled)))))))))+ (finally+ (.stop ai-server)))))(deftest feedback-endpoint-test(mt/with-premium-features #{:metabot-v3}diff --git a/enterprise/backend/test/metabase_enterprise/metabot_v3/client_test.clj b/enterprise/backend/test/metabase_enterprise/metabot_v3/client_test.cljindex 096fde38a89..d8d984dec38 100644--- a/enterprise/backend/test/metabase_enterprise/metabot_v3/client_test.clj+++ b/enterprise/backend/test/metabase_enterprise/metabot_v3/client_test.clj@@ -16,7 +16,7 @@[metabase.util.malli :as mu][toucan2.core :as t2])(:import- (java.io ByteArrayOutputStream PipedInputStream PipedOutputStream)+ (java.io ByteArrayOutputStream Closeable PipedInputStream PipedOutputStream)(metabase.server.streaming_response StreamingResponse)))(set! *warn-on-reflection* true)@@ -41,9 +41,12 @@(Thread/sleep ^long delay-ms))(finally(.close pipe))))- {:status 200- :headers {"content-type" "text/event-stream"}- :body ret})))+ {:status 200+ :headers {"content-type" "text/event-stream"}+ :body ret+ :http-client (reify Closeable+ (close [_this]+ (.close ret)))})))(defn consume-streaming-response"Execute a StreamingResponse and capture its output"diff --git a/test/metabase/server/streaming_response_test.clj b/test/metabase/server/streaming_response_test.cljindex b211a507248..21e6f89beb6 100644--- a/test/metabase/server/streaming_response_test.clj+++ b/test/metabase/server/streaming_response_test.clj@@ -17,6 +17,7 @@(:import(jakarta.servlet AsyncContext ServletOutputStream)(jakarta.servlet.http HttpServletResponse)+ (java.io Closeable InputStream)(java.util.concurrent Executors)(org.apache.commons.lang3.concurrent BasicThreadFactory$Builder)))@@ -157,7 +158,8 @@handler (fn [req respond _raise](respond(compojure.response/render- (streaming-response/streaming-response {:content-type "text/event-stream; charset=utf-8"} [os canceled-chan]+ (streaming-response/streaming-response {:content-type "text/event-stream; charset=utf-8"+ :headers {"Transfer-Encoding" "chunked"}} [os canceled-chan](try(loop [](if (a/poll! canceled-chan)@@ -182,15 +184,18 @@(let [res (http/request {:method :post :url url:as :stream:decompress-body false})]- (.read ^java.io.InputStream (:body res)) ;; start the handler- (.close ^java.io.Closeable (:body res))+ (.read ^InputStream (:body res)) ;; start the handler+ ;; NOTE: this is the gist here, calling .close on the body will consume request *completely*+ (.close ^Closeable (:http-client res))(u/poll {:thunk #(deref canceled):done? some?:interval-ms 5});; it's been 29 when I tested this, if it every becomes flaky maybe decrease the number?(is (< 20 @cnt) "Stopped writing when channel closed")- (testing "canceled-chan is working"- (is (= :nice @canceled))))))+ (testing "cancellation is working"+ ;; we're not checking for particular way of cancelling, because cancellation poll interval can conflict+ ;; with Thread/sleep and will make this test flaky+ (is (some? @canceled))))))(finally(.stop server)))))