The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Don't cut off Metabot chat turns at Jetty's async timeout (#83767)
Andrei Simionescu66acc3da5274
diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.ednindex a079b05f2d1..c60e820d041 100644--- a/.clj-kondo/config.edn+++ b/.clj-kondo/config.edn@@ -543,6 +543,7 @@metabase.settings.core/defsetting clj-kondo.lint-as/def-catch-allmetabase.settings.models.setting.multi-setting/define-multi-setting clojure.core/defmetabase.settings.models.setting/defsetting clj-kondo.lint-as/def-catch-all+ metabase.server.test-util/with-test-server clojure.core/letmetabase.sso.ldap/with-ldap-connection clojure.core/fnmetabase.sync.util/sum-for clojure.core/formetabase.sync.util/with-emoji-progress-bar clojure.core/letdiff --git a/src/metabase/cmd/resources/other-env-vars.md b/src/metabase/cmd/resources/other-env-vars.mdindex fb74faa8841..373573332ed 100644--- a/src/metabase/cmd/resources/other-env-vars.md+++ b/src/metabase/cmd/resources/other-env-vars.md@@ -283,7 +283,7 @@ Type: integer<br>Default: `600000`<br>Since: v35.0-Timeout of Jetty async threads, defined in milliseconds. The default is 10 minutes. Very few things might reach that timeout, since they return some type of data before, but things like CSV downloads might.+Timeout of Jetty async threads, defined in milliseconds. The default is 10 minutes. Very few things might reach that timeout, since they return some type of data before, but things like CSV downloads might. Metabot chat responses aren't bound by this timeout.### `MB_JETTY_DAEMON`diff --git a/src/metabase/metabot/api.clj b/src/metabase/metabot/api.cljindex f859b438f6c..a5055de9ad8 100644--- a/src/metabase/metabot/api.clj+++ b/src/metabase/metabot/api.clj@@ -205,7 +205,10 @@(let [enriched-context (metabot.context/create-context context {:metabot-id metabot-id:profile-id (keyword profile-id)})messages (concat history [message])]- (sr/streaming-response {:content-type "text/event-stream"} [^OutputStream os canceled-chan]+ ;; a reasoning model can spend longer than MB_JETTY_ASYNC_RESPONSE_TIMEOUT on one turn+ (sr/streaming-response {:content-type "text/event-stream"+ :async-timeout-ms (metabot.settings/metabot-chat-turn-async-timeout-ms)}+ [^OutputStream os canceled-chan](let [parts-atom (atom [])memory-atom (atom nil)canceled? (volatile! false)diff --git a/src/metabase/metabot/settings.clj b/src/metabase/metabot/settings.cljindex 1d3feca59a2..00a090f8f89 100644--- a/src/metabase/metabot/settings.clj+++ b/src/metabase/metabot/settings.clj@@ -17,6 +17,14 @@:export? false:doc false)+(defsetting metabot-chat-turn-async-timeout-ms+ (deferred-tru "Maximum duration of a Metabot chat turn in milliseconds.")+ :type :positive-integer+ :visibility :internal+ :default 1800000+ :encryption :no+ :export? false)+(defsetting metabot-enabled?(deferred-tru "Whether Metabot is enabled for regular usage."):type :booleandiff --git a/src/metabase/server/streaming_response.clj b/src/metabase/server/streaming_response.cljindex 2d33fa96255..5e11bcf8d73 100644--- a/src/metabase/server/streaming_response.clj+++ b/src/metabase/server/streaming_response.clj@@ -399,7 +399,7 @@(defn- respond[{:keys [^HttpServletResponse response ^AsyncContext async-context request-map response-map request]}- f {:keys [content-type status headers], :as options} finished-chan]+ f {:keys [content-type status headers async-timeout-ms], :as options} finished-chan](let [canceled-chan (a/promise-chan)completed? (AtomicBoolean. false)](.addListener async-context@@ -415,6 +415,8 @@(onComplete [_ _event])(onStartAsync [_ _event])))(try+ (when async-timeout-ms+ (.setTimeout async-context async-timeout-ms))(.setStatus response (or status 202))(let [gzip? (should-gzip-response? request-map)headers (cond-> (assoc (merge headers (:headers response-map))@@ -521,6 +523,8 @@of a query) must supply their own executor so they cannot exhaust it. The supplied executor's futures mustsupport real interruption (`Future.cancel(true)`) — the hung-request escalation interrupts the worker, and aForkJoinPool-backed executor silently ignores it.+ * `:async-timeout-ms` -- how long the response may run before Jetty completes it, cutting the stream short.+ Defaults to `MB_JETTY_ASYNC_RESPONSE_TIMEOUT`; `0` means no limit.* `:error-response-fn` -- optional function applied to any error body right before [[write-error!]] writes it to theclient. Usually added after the fact with [[with-error-response-fn]]."{:style/indent 2, :arglists '([options [os-binding canceled-chan-binding] & body])}diff --git a/test/metabase/metabot/api_test.clj b/test/metabase/metabot/api_test.cljindex 5b75da35e97..7e5088da81c 100644--- a/test/metabase/metabot/api_test.clj+++ b/test/metabase/metabot/api_test.clj@@ -26,9 +26,11 @@[metabase.metabot.test-util :as mut][metabase.premium-features.core :as premium-features][metabase.search.test-util :as search.tu]- [metabase.server.instance :as server.instance][metabase.server.streaming-response :as sr]+ [metabase.server.test-util :as server.tu][metabase.test :as mt]+ [metabase.test.http-client :as client]+ [metabase.test.server.handler :as test.server.handler][metabase.util :as u][metabase.util.json :as json][toucan2.core :as t2]))@@ -247,12 +249,8 @@:completion_tokens 50}}))(write! "data: [DONE]\n\n")))(catch Exception _e nil)))- req)))- llm-server- (doto (server.instance/create-server llm-handler {:port 0 :join? false})- .start)- llm-url (str "http://localhost:" (.. llm-server getURI getPort))]- (try+ req)))]+ (server.tu/with-test-server [llm-url llm-handler](mt/test-helpers-set-global-values!(search.tu/with-index-disabled;; the fake LLM server is on localhost, which the network policy refuses on a hosted instance@@ -302,9 +300,102 @@"the finalized turn is marked :finished? false — the cancel was detected")(is (= 2 (count (t2/select :model/MetabotMessage:conversation_id conversation-id)))- "start-turn! inserted exactly user + placeholder; no extra row from finalize"))))))))))- (finally- (.stop llm-server))))))+ "start-turn! inserted exactly user + placeholder; no extra row from finalize"))))))))))))))++(deftest turn-longer-than-async-response-timeout-finishes-test+ (testing "A turn that outlasts MB_JETTY_ASYNC_RESPONSE_TIMEOUT streams its finish event and finalizes as finished"+ (let [turn-started (promise)+ release (promise)+ finished (promise)+ slow-turn (reify clojure.lang.IReduceInit+ (reduce [_ rf init]+ (deliver turn-started true)+ (deref release 5000 nil)+ (reduce rf init [{:type :start :id "msg-1"} {:type :text :text "Done"}])))]+ (mt/with-temp-env-var-value! [mb-jetty-async-response-timeout 100]+ (mt/with-dynamic-fn-redefs+ [agent/run-agent-loop (constantly slow-turn)+ metabot.context/create-context (fn [ctx & _] ctx)+ metabot.self/context-window-tokens (constantly 1000)+ metabot.persistence/finalize-assistant-turn! (fn [_ _ & {:keys [finished?]}] (deliver finished finished?))]+ (let [handler (bound-fn [req respond _raise]+ (respond (compojure.response/render+ (if (= "/control" (:uri req))+ (sr/streaming-response {:content-type "text/plain"} [os _]+ (deref release 5000 nil)+ (.write os (.getBytes "done" "UTF-8")))+ (#'api/native-agent-streaming-request {:profile-id "internal"+ :message {:role "user" :content "Hi"}+ :context {}}))+ req)))]+ (server.tu/with-test-server [url handler]+ (try+ (let [turn (future (:body (server.tu/request url :get "/")))]+ (is (true? (deref turn-started 5000 ::timed-out)))+ (is (= "" (:body (server.tu/request url :get "/control")))+ "the timeout cuts a plain stream that started after the turn")+ (deliver release true)+ (let [lines (->> (deref turn 5000 "")+ str/split-lines+ (filter #(str/starts-with? % "data: ")))+ events (->> lines+ (remove #(= "data: [DONE]" %))+ (mapv #(json/decode+kw (subs % 6))))]+ (is (= "data: [DONE]" (last lines)))+ (is (= "finish" (:type (last events))))+ (is (true? (deref finished 5000 ::timed-out)))))+ (finally+ (deliver release true))))))))))++(deftest turn-through-real-routes-outlasts-async-response-timeout-test+ (testing "The real route and middleware let a long turn finish and persist the assistant as finished"+ (let [turn-started (promise)+ release (promise)+ slow-turn (reify clojure.lang.IReduceInit+ (reduce [_ rf init]+ (deliver turn-started true)+ (deref release 5000 nil)+ (reduce rf init [{:type :start :id "msg-1"} {:type :text :text "Done"}])))+ real-handler (test.server.handler/test-handler)+ handler (fn [req respond raise]+ (if (= "/control" (:uri req))+ (respond (compojure.response/render+ (sr/streaming-response {:content-type "text/plain"} [os _]+ (deref release 5000 nil)+ (.write os (.getBytes "done" "UTF-8")))+ req))+ (real-handler req respond raise)))]+ (mt/with-temp-env-var-value! [mb-jetty-async-response-timeout 100]+ (mt/with-temporary-setting-values [llm.settings/llm-providers llm.tu/default-connections+ metabot.settings/llm-metabot-provider test-provider+ metabot.settings/metabot-chat-turn-async-timeout-ms 2000]+ (mt/with-dynamic-fn-redefs [agent/run-agent-loop (constantly slow-turn)+ metabot.context/create-context (fn [ctx & _] ctx)+ metabot.self/context-window-tokens (constantly 1000)+ conversation-title/ensure-title! (constantly {:status :missing})]+ (mt/with-temp [:model/MetabotConversation {conversation-id :id} {:user_id (mt/user->id :rasta)}]+ (server.tu/with-test-server [url handler]+ (let [options (client/build-request-map (mt/user->credentials :rasta)+ {:message "Hi" :context {} :conversation_id conversation-id}+ nil)+ turn (future (server.tu/request url :post "/api/metabot/agent-streaming" options))]+ (try+ (is (true? (deref turn-started 5000 ::timed-out)))+ (is (= "" (:body (server.tu/request url :get "/control")))+ "the server timeout has elapsed while the turn is still running")+ (is (=? {:finished nil}+ (t2/select-one [:model/MetabotMessage :finished]+ :conversation_id conversation-id :role :assistant)))+ (deliver release true)+ (let [{:keys [status body]} (deref turn 5000 nil)]+ (is (= 202 status))+ (is (str/includes? (or body "") "\"type\":\"finish\""))+ (is (str/ends-with? (or body "") "data: [DONE]\n\n")))+ (is (true? (t2/select-one-fn :finished :model/MetabotMessage+ :conversation_id conversation-id :role :assistant)))+ (finally+ (deliver release true)+ (deref turn 5000 nil))))))))))))(deftest thrown-during-agent-setup-persists-as-errored-test(testing "A throwable escaping the agent loop (e.g. permission/setup throw beforediff --git a/test/metabase/server/streaming_response_test.clj b/test/metabase/server/streaming_response_test.cljindex d83043a7391..d6cddeb366c 100644--- a/test/metabase/server/streaming_response_test.clj+++ b/test/metabase/server/streaming_response_test.clj@@ -8,11 +8,11 @@[malli.error :as me][metabase.driver :as driver][metabase.query-processor.pipeline :as qp.pipeline]- [metabase.server.instance :as server.instance][metabase.server.protocols :as server.protocols][metabase.server.settings :as server.settings][metabase.server.streaming-response :as streaming-response][metabase.server.streaming-response.thread-pool :as thread-pool]+ [metabase.server.test-util :as server.tu][metabase.test :as mt][metabase.test.http-client :as client][metabase.util :as u]@@ -181,16 +181,11 @@(recur)))))(catch Exception _e(reset! canceled :not-nice))))- req)))- server (doto (server.instance/create-server handler {:port 0 :join? false})- .start)- url (str "http://localhost:" (.. server getURI getPort))]- (try+ req)))]+ (server.tu/with-test-server [url handler](with-redefs [streaming-response/async-cancellation-poll-interval-ms 5](testing "Closing body stops request handler"- (let [res (http/request {:method :post :url url- :as :stream- :decompress-body false})]+ (let [res (server.tu/request url :post "" {:as :stream})](.read ^InputStream (:body res)) ;; start the handler;; NOTE: this is the gist here, calling .close on the body will consume request *completely*(.close ^Closeable (:http-client res))@@ -202,9 +197,7 @@(testing "cancellation is working";; we're not checking for particular way of cancelling, because cancellation poll interval can conflict;; with Thread/sleep and will make this test flaky- (is (some? @canceled))))))- (finally- (.stop server)))))+ (is (some? @canceled)))))))))(deftest abort-on-committed-error-test(testing "An error after the response is committed aborts the connection so the client cannot read a complete body"@@ -216,24 +209,19 @@(.write os (.getBytes "a,b,c\n1,2,3\n" "UTF-8"))(.flush os)(streaming-response/write-error! os {:error "boom"} :csv 500))- req)))- server (doto (server.instance/create-server handler {:port 0 :join? false})- .start)- url (str "http://localhost:" (.. server getURI getPort))- consume (fn [] (let [res (http/request {:method :get, :url url, :as :stream, :decompress-body false})]- [res (slurp (:body res))]))]- (try- (testing "the response is chunked so a missing terminator is detectable"- ;; can't read the body cleanly, so just open a request to inspect the headers- (let [res (http/request {:method :get, :url url, :as :stream, :decompress-body false})]- (is (= "chunked" (get-in res [:headers "transfer-encoding"])))- (u/ignore-exceptions (.close ^InputStream (:body res)))))- (testing "consuming the whole body throws because the stream was aborted without a clean chunk terminator"- (is (thrown? Exception (consume))))- (testing "no JSON error blob is appended to the body"- (is (not (re-find #"boom" (try (second (consume)) (catch Exception _ ""))))))- (finally- (.stop server))))))+ req)))]+ (server.tu/with-test-server [url handler]+ (let [consume (fn [] (let [res (server.tu/request url :get "" {:as :stream})]+ [res (slurp (:body res))]))]+ (testing "the response is chunked so a missing terminator is detectable"+ ;; can't read the body cleanly, so just open a request to inspect the headers+ (let [res (server.tu/request url :get "" {:as :stream})]+ (is (= "chunked" (get-in res [:headers "transfer-encoding"])))+ (u/ignore-exceptions (.close ^InputStream (:body res)))))+ (testing "consuming the whole body throws because the stream was aborted without a clean chunk terminator"+ (is (thrown? Exception (consume))))+ (testing "no JSON error blob is appended to the body"+ (is (not (re-find #"boom" (try (second (consume)) (catch Exception _ "")))))))))))(def ^:private ^:dynamic *number-of-cans* nil)@@ -766,6 +754,47 @@(is (false? (.await complete-called 100 TimeUnit/MILLISECONDS))"Worker thread should not call .complete when timeout already completed the context"))))))+(deftest async-timeout-ms-test+ (mt/with-temp-env-var-value! [mb-jetty-async-response-timeout 100]+ (let [started (promise)+ release (promise)+ handler (fn [req respond _raise]+ (let [no-timeout? (= "/no-timeout" (:uri req))+ options (cond-> {:content-type "text/plain"}+ no-timeout? (assoc :async-timeout-ms 0))]+ (respond+ (compojure.response/render+ (streaming-response/streaming-response options [os _]+ (when no-timeout?+ (deliver started true))+ (deref release 5000 nil)+ (.write os (.getBytes "done" "UTF-8")))+ req))))]+ (server.tu/with-test-server [url handler]+ (try+ (let [no-timeout (future (:body (server.tu/request url :get "/no-timeout")))]+ (is (true? (deref started 5000 ::timed-out)))+ (testing "Jetty completes a response that runs past MB_JETTY_ASYNC_RESPONSE_TIMEOUT"+ (is (= "" (:body (server.tu/request url :get "/")))))+ (deliver release true)+ (testing "unless the response sets :async-timeout-ms 0"+ (is (= "done" (deref no-timeout 5000 ::timed-out)))))+ (finally+ (deliver release true)))))))++(deftest positive-async-timeout-ms-test+ (testing "A positive response timeout replaces the shorter server timeout"+ (mt/with-temp-env-var-value! [mb-jetty-async-response-timeout 100]+ (let [handler (fn [req respond _raise]+ (respond+ (compojure.response/render+ (streaming-response/streaming-response {:content-type "text/plain" :async-timeout-ms 2000} [os _]+ (a/<!! (a/timeout 300))+ (.write os (.getBytes "done" "UTF-8")))+ req)))]+ (server.tu/with-test-server [url handler]+ (is (= "done" (:body (server.tu/request url :get "/")))))))))+(deftest do-f-async-custom-executor-test(testing "the :executor option runs `f` on that executor instead of the shared streaming pool"(let [executor (java.util.concurrent.Executors/newSingleThreadExecutordiff --git a/test/metabase/server/test_util.clj b/test/metabase/server/test_util.cljnew file mode 100644index 00000000000..4f59a6d06e8--- /dev/null+++ b/test/metabase/server/test_util.clj@@ -0,0 +1,33 @@+(ns metabase.server.test-util+ (:require+ [clj-http.client :as http]+ [metabase.server.instance :as server.instance]))++(set! *warn-on-reflection* true)++(defn do-with-test-server+ "Call `f` with the base URL of a temporary Jetty server running `handler`, then stop the server."+ [handler f]+ (let [server (server.instance/create-server (bound-fn [request respond raise]+ (handler request respond raise))+ {:port 0 :join? false})]+ (try+ (.start server)+ (f (str "http://localhost:" (.. server getURI getPort)))+ (finally+ (.stop server)))))++(defmacro with-test-server+ "Run `body` with `url` bound to the base URL of a temporary Jetty server running `handler`."+ [[url handler] & body]+ `(do-with-test-server ~handler (fn [~url] ~@body)))++(defn request+ "Make an HTTP request to `path` on a test server, with optional clj-http `options`."+ [url method path & [options]]+ (http/request (merge {:url (str url path)+ :method method+ :decompress-body false+ :connection-timeout 5000+ :socket-timeout 5000}+ options)))