The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Drop the upstream LLM connection when a Metabot stream closes (#85023)
Paolo Rascunà67b958b8857f
diff --git a/src/metabase/metabot/self/adapter.clj b/src/metabase/metabot/self/adapter.cljindex a4702755c73..56d0eacea63 100644--- a/src/metabase/metabot/self/adapter.clj+++ b/src/metabase/metabot/self/adapter.clj@@ -406,7 +406,9 @@depends on the connection rather than only on the response. Applies to bothphases below.:read-stream - turns the response body into a reducible of the provider's events, for a stream- that is not SSE. Defaults to [[core/sse-reducible]].+ that is not SSE. Defaults to [[core/sse-reducible]]. When the request has its own+ client, a close of the body it gets drops the connection and does not read+ the rest of the body (see [[core/abort-on-close]]).:wrap-stream - applied to the reducible before error translation, for an adapter with its owntranslation to do first.:on-request-error - replaces the default [[rethrow!]] catch, for an adapter that retries. Only@@ -445,7 +447,8 @@;; encoded up front, since a provider may sign over the body:body (json/encode body)}request-options)- :body+ ;; on this thread, right after the request, so it sees the connection manager clj-http saw+ (core/abort-on-close request-options)read-stream(debug/capture-stream {:provider slug:model modeldiff --git a/src/metabase/metabot/self/core.clj b/src/metabase/metabot/self/core.cljindex 6beb76fa1d8..4d84b2af858 100644--- a/src/metabase/metabot/self/core.clj+++ b/src/metabase/metabot/self/core.clj@@ -1,6 +1,7 @@(ns metabase.metabot.self.core(:require[clj-http.client :as http]+ [clj-http.conn-mgr :as conn][clojure.java.io :as io][clojure.string :as str][clojure.walk :as walk]@@ -21,7 +22,7 @@[metabase.util.malli.schema :as ms][metabase.util.o11y :refer [with-span]])(:import- (java.io BufferedReader Closeable InputStream)+ (java.io BufferedReader Closeable FilterInputStream InputStream IOException)(java.util.concurrent Callable Executors ExecutorService)))(set! *warn-on-reflection* true)@@ -357,6 +358,54 @@(close [_](.close input))))+(defn- close-released-body!+ "Close a streamed `body` whose client is already closed. Ignore the error that this close is known to throw,+ and log it at debug level."+ [^InputStream body]+ ;; The body close releases the connection to a manager that is shut down. Apache HttpClient then throws+ ;; `IllegalStateException: Connection not obtained from this manager`, or an `IOException` from the closed+ ;; socket. See https://github.com/dakrone/clj-http/issues/627+ (try+ (.close body)+ (catch IllegalStateException e+ (log/debug e "Ignored the error from the body close after the client close"))+ (catch IOException e+ (log/debug e "Ignored the error from the body close after the client close"))))++(defn- own-client?+ "True when clj-http made the client of a request for that request only.++ A bound or passed connection manager, or a passed client, can serve other requests."+ [request-options]+ (and (nil? conn/*connection-manager*)+ (nil? (:connection-manager request-options))+ (nil? (:http-client request-options))))++(defn abort-on-close+ "Return the body of a streamed clj-http `response`, changed so that a close drops the connection.++ A close of the returned body closes the request's own `:http-client` first, so the provider sees a disconnect.+ Then it closes the body, and it does not read the rest of the body. Over TLS 1.3 the client close can wait for+ the next byte from the provider, up to the socket timeout.++ Returns `:body` unchanged when it is not an `InputStream`, when the response has no `Closeable` `:http-client`,+ or when `request-options` (the clj-http options of the request) share a client or a connection manager with+ other requests. Call it on the thread that made the request, so it sees the same bound connection manager.+ This is the workaround from https://github.com/dakrone/clj-http/issues/627."+ [{:keys [body http-client]} request-options]+ (if (and (instance? InputStream body)+ (instance? Closeable http-client)+ (own-client? request-options))+ ;; Without this wrapper, a close of a chunked body reads the rest of the body. It returns only when the provider+ ;; finishes, and the provider continues to generate until then.+ (proxy [FilterInputStream] [body]+ (close []+ (try+ (.close ^Closeable http-client)+ (finally+ (close-released-body! body)))))+ body))+;;; AISDK5(def finish-reasonsdiff --git a/test/metabase/metabot/api_test.clj b/test/metabase/metabot/api_test.cljindex e5a508271c6..9120706bcdf 100644--- a/test/metabase/metabot/api_test.clj+++ b/test/metabase/metabot/api_test.clj@@ -463,6 +463,112 @@^String [data](str "data: " (json/encode data) "\n\n"))+(defn- fake-openrouter-handler+ "A fake OpenRouter (Chat Completions) server: an async ring handler that streams SSE text deltas, one every+ 10 ms, while `cnt` is positive. It decrements `cnt` after each delta, and stops when its consumer disconnects.++ When it stops with `cnt` still positive, it delivers the remaining `cnt` to the optional `:stopped-early` promise."+ [cnt & {:keys [stopped-early]}]+ (let [chat-id (str "chatcmpl-" (random-uuid))]+ (fn [req respond _raise]+ (respond+ (compojure.response/render+ (sr/streaming-response {:content-type "text/event-stream; charset=utf-8"} [os llm-canceled-chan]+ (try+ (let [write! (fn [^String s]+ (.write os (.getBytes s "UTF-8"))+ (.flush os))+ canceled? #(some? (a/poll! llm-canceled-chan))]+ ;; First chunk: role assignment (empty content to establish assistant role)+ (write! (sse-event {:id chat-id+ :model "anthropic/claude-haiku-4-5"+ :choices [{:index 0+ :delta {:role "assistant" :content ""}+ :finish_reason nil}]}))+ ;; Stream text content chunks slowly, stop early if the consumer disconnects+ (loop []+ (when (and (pos? @cnt) (not (canceled?)))+ (write! (sse-event {:id chat-id+ :model "anthropic/claude-haiku-4-5"+ :choices [{:index 0+ ;; pad past Jetty's 8 KB buffer so .flush reaches the client mid-stream+ :delta {:content (str "chunk-" @cnt " " (apply str (repeat 512 \x)))}+ :finish_reason nil}]}))+ (swap! cnt dec)+ (Thread/sleep 10)+ (recur)))+ (when-not (canceled?)+ ;; Finish reason+ (write! (sse-event {:id chat-id+ :model "anthropic/claude-haiku-4-5"+ :choices [{:index 0+ :delta {}+ :finish_reason "stop"}]}))+ ;; Usage (separate final chunk, as OpenRouter does)+ (write! (sse-event {:id chat-id+ :model "anthropic/claude-haiku-4-5"+ :choices []+ :usage {:prompt_tokens 10+ :completion_tokens 50}}))+ (write! "data: [DONE]\n\n")))+ (catch Exception _e nil)+ ;; A write after the disconnect can throw before `canceled?` turns true. So the `finally` reports the+ ;; early stop, which covers both paths.+ (finally+ (when (and stopped-early (pos? @cnt))+ (deliver stopped-early @cnt)))))+ req)))))++(defn- do-with-closed-agent-stream!+ "Start a native-agent turn against the fake LLM at `llm-url`, read one byte of the response, and close the+ client. Wait up to 3 s for `finalize-assistant-turn!`, then call `f` with `:stored-parts`, `:stored-kwargs`+ (the finalize arguments, nil if it did not run) and the `:conversation-id`."+ [llm-url f]+ (let [stored-parts (atom nil)+ stored-kwargs (atom nil)]+ (mt/test-helpers-set-global-values!+ (search.tu/with-index-disabled+ ;; the fake LLM server is on localhost, which the network policy refuses on a hosted instance+ (mt/with-temp-env-var-value! [mb-llm-allowed-networks "allow-all"]+ (mt/with-temporary-setting-values [llm.settings/llm-providers [(llm.tu/connection "openrouter" {:base-url llm-url})]+ metabot.settings/llm-metabot-provider test-provider]+ (let [real-llm-request self.core/request]+ (with-redefs [scope/resolve-user-permissions (constantly scope/all-yes-permissions)+ conversation-title/ensure-title! (constantly {:status :missing})+ ;; The fake LLM server gzips whenever the caller accepts it, and clj-http+ ;; wraps the body in a GZIPInputStream. Closing mid-stream causes ZLIB errors.+ self.core/request (fn [auth req]+ (real-llm-request auth (assoc req :decompress-body false)))+ metabot.context/create-context (fn [ctx & _] ctx)+ metabot.persistence/finalize-assistant-turn! (fn [_pk parts & kwargs]+ (reset! stored-parts parts)+ (reset! stored-kwargs (apply hash-map kwargs)))+ sr/async-cancellation-poll-interval-ms 5]+ (mt/with-model-cleanup [:model/MetabotMessage+ [:model/MetabotConversation :created_at]]+ (let [conversation-id (str (random-uuid))+ response (mt/user-real-request-full-response+ :rasta :post 202 "metabot/agent-streaming"+ {:request-options {:as :stream+ :decompress-body false}}+ {:message "Test closure"+ :context {}+ :conversation_id conversation-id+ :state {}})]+ (.read ^java.io.InputStream (:body response)) ;; start the handler+ ;; Close the underlying client, not the body stream: closing the body would+ ;; make clj-http drain the (now chunked) response to completion, which looks+ ;; like a normal finish rather than a disconnect. Closing the client aborts+ ;; the connection, which is what the server's cancel loop detects.+ (.close ^java.io.Closeable (:http-client response))+ (u/poll {:thunk #(deref stored-parts)+ :done? some?+ :interval-ms 10+ :timeout-ms 3000})+ (f {:stored-parts @stored-parts+ :stored-kwargs @stored-kwargs+ :conversation-id conversation-id})))))))))))+(deftest closing-connection-native-agent-test(testing "When the client closes a native-agent streaming connection, thepipeline tears down and the partial turn is persisted as aborted."@@ -476,107 +582,41 @@;; the agent loop unwinds, and `finalize-assistant-turn!` is called from the;; `finally` with `:finished? false`, UPDATEing the placeholder row inserted;; by `start-turn!`.- (let [total-chunks 30+ (let [cnt (atom 30)]+ (server.tu/with-test-server [llm-url (fake-openrouter-handler cnt)]+ (testing "Closing stream body tears down the pipeline and persists the aborted turn"+ (do-with-closed-agent-stream!+ llm-url+ (fn [{:keys [stored-parts stored-kwargs conversation-id]}]+ (is (some? stored-parts)+ "finalize-assistant-turn! was called even though the client disconnected")+ (is (false? (:finished? stored-kwargs))+ "the finalized turn is marked :finished? false — the cancel was detected")+ (is (= 2 (count (t2/select :model/MetabotMessage+ :conversation_id conversation-id)))+ "start-turn! inserted exactly user + placeholder; no extra row from finalize"))))))))++(deftest closing-connection-stops-upstream-llm-test+ (testing "When the client closes a native-agent streaming connection during a long LLM stream, Metabase drops+ its connection to the LLM, so the LLM stops sending and the turn finalizes at once"+ ;; Before the workaround for https://github.com/dakrone/clj-http/issues/627, a close of the LLM response body+ ;; read the rest of the chunked stream. The turn then finalized only when the LLM finished (about 20 s here).+ (let [total-chunks 2000cnt (atom total-chunks)- stored-parts (atom nil)- stored-kwargs (atom nil)- chat-id (str "chatcmpl-" (random-uuid))- ;; Fake OpenRouter API: streams Chat Completions SSE text deltas.- llm-handler- (fn [req respond _raise]- (respond- (compojure.response/render- (sr/streaming-response {:content-type "text/event-stream; charset=utf-8"} [os llm-canceled-chan]- (try- (let [write! (fn [^String s]- (.write os (.getBytes s "UTF-8"))- (.flush os))- canceled? #(some? (a/poll! llm-canceled-chan))]- ;; First chunk: role assignment (empty content to establish assistant role)- (write! (sse-event {:id chat-id- :model "anthropic/claude-haiku-4-5"- :choices [{:index 0- :delta {:role "assistant" :content ""}- :finish_reason nil}]}))- ;; Stream text content chunks slowly, stop early if the consumer disconnects- (loop []- (when (and (pos? @cnt) (not (canceled?)))- (write! (sse-event {:id chat-id- :model "anthropic/claude-haiku-4-5"- :choices [{:index 0- ;; pad past Jetty's 8 KB buffer so .flush reaches the client mid-stream- :delta {:content (str "chunk-" @cnt " " (apply str (repeat 512 \x)))}- :finish_reason nil}]}))- (swap! cnt dec)- (Thread/sleep 10)- (recur)))- (when-not (canceled?)- ;; Finish reason- (write! (sse-event {:id chat-id- :model "anthropic/claude-haiku-4-5"- :choices [{:index 0- :delta {}- :finish_reason "stop"}]}))- ;; Usage (separate final chunk, as OpenRouter does)- (write! (sse-event {:id chat-id- :model "anthropic/claude-haiku-4-5"- :choices []- :usage {:prompt_tokens 10- :completion_tokens 50}}))- (write! "data: [DONE]\n\n")))- (catch Exception _e nil)))- req)))]- (server.tu/with-test-server [llm-url llm-handler]- (mt/test-helpers-set-global-values!- (search.tu/with-index-disabled- ;; the fake LLM server is on localhost, which the network policy refuses on a hosted instance- (mt/with-temp-env-var-value! [mb-llm-allowed-networks "allow-all"]- (mt/with-temporary-setting-values [llm.settings/llm-providers [(llm.tu/connection "openrouter" {:base-url llm-url})]- metabot.settings/llm-metabot-provider test-provider]- (let [real-llm-request self.core/request]- (with-redefs [scope/resolve-user-permissions (constantly scope/all-yes-permissions)- conversation-title/ensure-title! (constantly {:status :missing})- ;; The fake LLM server gzips whenever the caller accepts it, and clj-http- ;; wraps the body in a GZIPInputStream. Closing mid-stream causes ZLIB errors.- self.core/request (fn [auth req]- (real-llm-request auth (assoc req :decompress-body false)))- metabot.context/create-context (fn [ctx & _] ctx)- metabot.persistence/finalize-assistant-turn! (fn [_pk parts & kwargs]- (reset! stored-parts parts)- (reset! stored-kwargs (apply hash-map kwargs)))- sr/async-cancellation-poll-interval-ms 5]- (testing "Closing stream body tears down the pipeline and persists the aborted turn"- (reset! cnt total-chunks)- (reset! stored-parts nil)- (reset! stored-kwargs nil)- (mt/with-model-cleanup [:model/MetabotMessage- [:model/MetabotConversation :created_at]]- (let [conversation-id (str (random-uuid))- response (mt/user-real-request-full-response- :rasta :post 202 "metabot/agent-streaming"- {:request-options {:as :stream- :decompress-body false}}- {:message "Test closure"- :context {}- :conversation_id conversation-id- :state {}})]- (.read ^java.io.InputStream (:body response)) ;; start the handler- ;; Close the underlying client, not the body stream: closing the body would- ;; make clj-http drain the (now chunked) response to completion, which looks- ;; like a normal finish rather than a disconnect. Closing the client aborts- ;; the connection, which is what the server's cancel loop detects.- (.close ^java.io.Closeable (:http-client response))- (u/poll {:thunk #(deref stored-parts)- :done? some?- :interval-ms 10- :timeout-ms 3000})- (is (some? @stored-parts)- "finalize-assistant-turn! was called even though the client disconnected")- (is (false? (:finished? @stored-kwargs))- "the finalized turn is marked :finished? false — the cancel was detected")- (is (= 2 (count (t2/select :model/MetabotMessage- :conversation_id conversation-id)))- "start-turn! inserted exactly user + placeholder; no extra row from finalize"))))))))))))))+ stopped-early (promise)]+ (server.tu/with-test-server [llm-url (fake-openrouter-handler cnt :stopped-early stopped-early)]+ (do-with-closed-agent-stream!+ llm-url+ (fn [{:keys [stored-parts stored-kwargs]}]+ (is (some? stored-parts)+ "finalize-assistant-turn! ran within 3 s of the disconnect")+ (is (false? (:finished? stored-kwargs))+ "the finalized turn is marked :finished? false")+ (let [remaining (deref stopped-early 2000 nil)]+ (is (some? remaining)+ "the fake LLM saw the disconnect and stopped before it sent all its chunks")+ (is (< (- total-chunks (or remaining 0)) 300)+ "the fake LLM saw the disconnect long before it sent all its chunks"))))))))(deftest turn-longer-than-async-response-timeout-finishes-test(testing "A turn that outlasts MB_JETTY_ASYNC_RESPONSE_TIMEOUT streams its finish event and finalizes as finished"diff --git a/test/metabase/metabot/self/adapter_test.clj b/test/metabase/metabot/self/adapter_test.cljindex 94c666684d1..2e94b4f34b2 100644--- a/test/metabase/metabot/self/adapter_test.clj+++ b/test/metabase/metabot/self/adapter_test.clj@@ -1,6 +1,7 @@(ns metabase.metabot.self.adapter-test(:require[clj-http.client :as http]+ [clj-http.conn-mgr :as conn][clojure.edn :as edn][clojure.string :as str][clojure.test :refer [deftest is testing]]@@ -23,9 +24,12 @@[metabase.metabot.self.vllm :as vllm][metabase.metabot.self.xai :as xai][metabase.metabot.self.zai :as zai]+ [metabase.metabot.test-util :as mut][metabase.premium-features.core :as premium-features][metabase.test :as mt]- [metabase.util.malli.registry :as mr]))+ [metabase.util.malli.registry :as mr])+ (:import+ (java.io ByteArrayInputStream Closeable InputStream IOException)))(set! *warn-on-reflection* true)@@ -379,3 +383,256 @@(is (= ["From name"](names zai/list-models{:data [{:id "glm-5.2" :name "From name" :display_name "From display_name"}]}))))))++;;; ──────────────────────────────────────────────────────────────────+;;; Upstream connection on close+;;;+;;; A close of a clj-http `:as :stream` body reads the rest of a chunked body before it returns, so the+;;; provider continues to generate: https://github.com/dakrone/clj-http/issues/627. These tests check that a+;;; stream from [[adapter/stream!]] drops the connection when its body closes.+;;; ──────────────────────────────────────────────────────────────────++(defn- timed-reduce+ "Call `thunk`. Return its `:result`, the exception it threw as `:thrown`, and the `:elapsed-ms`."+ [thunk]+ (let [start (System/nanoTime)+ [result thrown] (try+ [(thunk) nil]+ (catch Throwable e+ [nil e]))]+ {:result result+ :thrown thrown+ :elapsed-ms (quot (- (System/nanoTime) start) 1000000)}))++(defn- with-chunked-server!+ "Call `(f srv)` with a fresh [[mut/start-chunked-sse-server!]] made from `opts`, with localhost allowed. Stop the+ server after."+ [opts f]+ (let [srv (mut/start-chunked-sse-server! opts)]+ (try+ (mt/with-temp-env-var-value! [mb-llm-allowed-networks "allow-all"]+ (f srv))+ (finally+ ((:stop! srv))))))++(defn- server-request-opts+ [srv]+ {:model "fake-model"+ :input [{:role :user :content "hi"}]+ :credentials {:api-key "sk-or-test" :base-url (:url srv)}})++(defn- openrouter-stream+ [srv]+ (openrouter/openrouter-raw (server-request-opts srv)))++(defn- take-events+ "A reducing fn that collects events, and stops after `n`."+ [n]+ (fn [acc event]+ (let [acc (conj acc event)]+ (if (= n (count acc))+ (reduced acc)+ acc))))++(defn- disconnected-count+ "The events the server sent before it saw the disconnect, or nil if it saw none within 2 s."+ [srv]+ (deref (:disconnected srv) 2000 nil))++(defn- dropped-early?+ "True when the server saw the disconnect after fewer than `n` events."+ [srv n]+ (< (or (disconnected-count srv) ##Inf) n))++(defn- cause-chain+ [e]+ (take-while some? (iterate ex-cause e)))++(deftest stream-close-drops-upstream-connection-test+ (testing "a reduction that stops early drops the upstream connection, and does not read the rest of the body"+ (with-chunked-server! {:total 2000 :interval-ms 10}+ (fn [srv]+ (let [{:keys [result thrown elapsed-ms]} (timed-reduce #(reduce (take-events 3) [] (openrouter-stream srv)))]+ (is (nil? thrown))+ (is (= 3 (count result)))+ (is (< elapsed-ms 2000))+ (is (dropped-early? srv 100)))))))++(deftest stream-done-before-eof-drops-upstream-connection-test+ (testing "a stream that ends at `[DONE]` drops the connection, also when the server sends more bytes after it"+ (with-chunked-server! {:total 3 :after :keep-alive}+ (fn [srv]+ (let [{:keys [result thrown elapsed-ms]} (timed-reduce #(reduce conj [] (openrouter-stream srv)))]+ (is (nil? thrown))+ (is (= 3 (count result)))+ (is (< elapsed-ms 2000))+ (is (some? (disconnected-count srv))))))))++(deftest stream-gzip-close-drops-upstream-connection-test+ (testing "a gzip body, which clj-http decompresses, also drops the connection when it closes"+ (with-chunked-server! {:total 2000 :interval-ms 10 :gzip? true}+ (fn [srv]+ (let [{:keys [result thrown elapsed-ms]} (timed-reduce #(reduce (take-events 3) [] (openrouter-stream srv)))]+ (is (nil? thrown))+ (is (= 3 (count result)))+ (is (< elapsed-ms 2000))+ (is (dropped-early? srv 100)))))))++(defn- read-one-buffer+ "A `:read-stream` like Bedrock's `runtime-events`: its reduction opens `body` with `with-open`, reads one buffer+ and returns `init`."+ [^InputStream body]+ (reify clojure.lang.IReduceInit+ (reduce [_ _rf init]+ (with-open [in body]+ (.read in (byte-array 8192))+ init))))++(deftest stream-custom-read-stream-drops-upstream-connection-test+ (testing "a `:read-stream` that is not SSE, and closes the body itself, also drops the connection"+ (with-chunked-server! {:total 2000 :interval-ms 10}+ (fn [srv]+ (let [{:keys [thrown elapsed-ms]} (timed-reduce+ #(reduce conj []+ (adapter/stream! @#'openrouter/provider+ (server-request-opts srv)+ {:path "/v1/chat/completions"+ :body {}+ :read-stream read-one-buffer})))]+ (is (nil? thrown))+ (is (< elapsed-ms 2000))+ (is (dropped-early? srv 100)))))))++(defn- recording-body+ "An `InputStream` over `text` that adds `:body` to `log` when it closes, and then throws `close-ex` if given."+ ^InputStream [log ^String text close-ex]+ (proxy [ByteArrayInputStream] [(.getBytes text "UTF-8")]+ (close []+ (swap! log conj :body)+ (when close-ex+ (throw close-ex)))))++(defn- recording-client+ "A `Closeable` that adds `:client` to `log` when it closes."+ [log]+ (reify Closeable+ (close [_]+ (swap! log conj :client))))++(def ^:private stub-sse+ "data: {\"n\":1}\n\ndata: {\"n\":2}\n\ndata: [DONE]\n\n")++(def ^:private stub-request-opts+ {:model "fake-model"+ :input [{:role :user :content "hi"}]+ :credentials {:api-key "sk-or-test" :base-url "https://openrouter.ai/api"}})++(defn- stub-stream!+ "Reduce [[adapter/stream!]] to the end, with `http/request` stubbed to return `response`. `stream-opts` merge+ into the minimal stream opts."+ [response stream-opts]+ (with-redefs [http/request (fn [_] (assoc response :status 200))]+ (timed-reduce #(reduce conj []+ (adapter/stream! @#'openrouter/provider+ stub-request-opts+ (merge {:path "/v1/chat/completions" :body {}} stream-opts))))))++(defn- recorded-close-stream!+ "[[stub-stream!]] over a recording body that throws `close-ex` on close, with a recording client. Adds the+ close log to the result as `:closed`."+ [close-ex stream-opts]+ (let [log (atom [])]+ (assoc (stub-stream! {:body (recording-body log stub-sse close-ex)+ :http-client (recording-client log)}+ stream-opts)+ :closed @log)))++(deftest stream-close-order-test+ (testing "the client closes before the body, so the body close does not read the rest of the stream"+ (let [{:keys [result thrown closed]} (recorded-close-stream! nil nil)]+ (is (nil? thrown))+ (is (= [{:n 1} {:n 2}] result))+ (is (= [:client :body] closed)))))++(deftest stream-close-ignores-released-connection-error-test+ (testing "the error that the body close throws after the client closes is ignored"+ (testing "IllegalStateException"+ (let [{:keys [result thrown]} (recorded-close-stream!+ (IllegalStateException. "Connection not obtained from this manager")+ nil)]+ (is (nil? thrown))+ (is (= [{:n 1} {:n 2}] result))))+ (testing "IOException"+ (let [{:keys [result thrown]} (recorded-close-stream! (IOException. "Socket closed") nil)]+ (is (nil? thrown))+ (is (= [{:n 1} {:n 2}] result))))))++(deftest stream-close-keeps-other-close-errors-test+ (testing "any other error from the body close reaches the caller as a provider error"+ (let [boom (RuntimeException. "boom")+ {:keys [thrown]} (recorded-close-stream! boom nil)]+ (is (= :provider-request-failed (:error-code (ex-data thrown))))+ (is (identical? boom (ex-cause thrown))))))++(deftest stream-rf-exception-survives-close-test+ (testing "an exception from `rf` reaches the caller; the close error does not replace it"+ (with-chunked-server! {:total 300 :interval-ms 10}+ (fn [srv]+ (let [rf (fn [acc event]+ (when (= 1 (count acc))+ (throw (ex-info "boom" {::boom true})))+ (conj acc event))+ {:keys [thrown elapsed-ms]} (timed-reduce #(reduce rf [] (openrouter-stream srv)))+ chain (cause-chain thrown)]+ (is (some (comp ::boom ex-data) chain))+ (is (not-any? #(instance? IllegalStateException %) chain))+ (is (< elapsed-ms 2000))+ (is (dropped-early? srv 100)))))))++(deftest stream-normal-finish-raises-no-error-test+ (testing "a stream that reads to its end gives every event, no exception and no warning"+ (doseq [after [:done :eof]]+ (testing after+ (with-chunked-server! {:total 5 :after after}+ (fn [srv]+ (mt/with-log-messages-for-level [messages [metabase.metabot.self :warn]]+ (let [{:keys [result thrown]} (timed-reduce #(reduce conj [] (openrouter-stream srv)))]+ (is (nil? thrown))+ (is (= 5 (count result)))+ (is (empty? (messages)))))))))))++(deftest stream-close-error-logged-at-debug-test+ (testing "a stream that ends at `[DONE]` logs the ignored body-close error at debug level"+ (with-chunked-server! {:total 5 :after :done}+ (fn [srv]+ (mt/with-log-messages-for-level [messages [metabase.metabot.self.core :debug]]+ (let [{:keys [thrown]} (timed-reduce #(reduce conj [] (openrouter-stream srv)))]+ (is (nil? thrown))+ (is (=? [{:level :debug+ :message "Ignored the error from the body close after the client close"}]+ (messages)))))))))++(deftest stream-keeps-shared-client-test+ (testing "a client that can serve other requests stays open; only the body closes"+ (let [cm (conn/make-reusable-conn-manager {})]+ (try+ (testing "a bound connection manager"+ (binding [conn/*connection-manager* cm]+ (is (= [:body] (:closed (recorded-close-stream! nil nil))))))+ (testing "a connection manager in the request options"+ (is (= [:body] (:closed (recorded-close-stream! nil {:request-options {:connection-manager cm}})))))+ (testing "an http client in the request options"+ (is (= [:body] (:closed (recorded-close-stream! nil {:request-options {:http-client (recording-client (atom []))}})))))+ (finally+ (conn/shutdown-manager cm))))))++(deftest stream-passes-through-stub-bodies-test+ (testing "a response the helper cannot abort reaches `read-stream` as the identical body"+ (doseq [[label response] [["a map body" {:body {:a 1}}]+ ["a stream with no client" {:body (ByteArrayInputStream. (byte-array 0))}]+ ["a client that is not Closeable" {:body (ByteArrayInputStream. (byte-array 0))+ :http-client (Object.)}]]]+ (testing label+ (let [seen (atom nil)]+ (stub-stream! response {:read-stream (fn [body] (reset! seen body) [])})+ (is (identical? (:body response) @seen)))))))diff --git a/test/metabase/metabot/self_test.clj b/test/metabase/metabot/self_test.cljindex ba78b10d171..3c3b67f4a2e 100644--- a/test/metabase/metabot/self_test.clj+++ b/test/metabase/metabot/self_test.clj@@ -39,8 +39,7 @@[metabase.util.json :as json][metabase.util.log.capture :as log.capture][metabase.util.malli :as mu]- [metabase.util.malli.registry :as mr]- [ring.adapter.jetty :as jetty]))+ [metabase.util.malli.registry :as mr]))(set! *warn-on-reflection* true)@@ -428,48 +427,6 @@(self.core/sse-reducible istream))](is (= ["msg-0" "msg-1" "msg-2"] result)))))-(deftest sse-reducible-stops-response-test- (let [cnt (atom 30)- handler (fn [_req]- (let [out (java.io.PipedOutputStream.)- in (java.io.PipedInputStream. out 65536)]- (future- (try- (dotimes [i @cnt]- (.write out (.getBytes (test-util/make-sse [(str "msg-" i)])))- (.flush out)- (swap! cnt dec)- (Thread/sleep 10))- (catch Exception _)- (finally- (try (.close out)- (catch Exception _)))))- {:status 200- :headers {"Content-Type" "text/event-stream"- "Cache-Control" "no-cache"}- :body in}))- server (jetty/run-jetty handler {:port 0- :join? false- :output-buffer-size 1})- url (str "http://localhost:" (.. server getURI getPort))]- (try- (let [res (http/request {:method :post :url url :as :stream})- reducible (self.core/sse-reducible (:body res))- ;; Take only 2 items, which should trigger early termination- result (reduce (fn [acc item]- (if (< (count acc) 2)- (conj acc item)- (reduced acc)))- []- reducible)]- (is (= ["msg-0" "msg-1"] result))- ;; Give time for cancellation to propagate- (Thread/sleep 30)- ;; Should have stopped writing when reduction stopped- (is (> @cnt 20) "SHOULD have stopped writing when reduction terminated early"))- (finally- (.stop server)))))-(deftest lite-aisdk-xf-test(testing "streams text deltas immediately instead of batching"(let [chunks [{:type :start :messageId "msg-1"}diff --git a/test/metabase/metabot/test_util.clj b/test/metabase/metabot/test_util.cljindex fa638f4fac6..50da4f006be 100644--- a/test/metabase/metabot/test_util.clj+++ b/test/metabase/metabot/test_util.clj@@ -6,8 +6,13 @@[metabase.metabot.tools :as metabot.tools][metabase.metabot.tools.core :as tools.core][metabase.metabot.tools.error :as tools.error]+ [metabase.util :as u][metabase.util.json :as json]- [metabase.util.log :as log]))+ [metabase.util.log :as log])+ (:import+ (java.io ByteArrayOutputStream InputStream IOException OutputStream)+ (java.net InetAddress ServerSocket Socket SocketException)+ (java.util.zip GZIPOutputStream)))(set! *warn-on-reflection* true)@@ -229,3 +234,222 @@:display-name "snip":snippet-id Integer/MAX_VALUE}}}{:lib/type :mbql.stage/mbql}]})++;;; ──────────────────────────────────────────────────────────────────+;;; Chunked SSE fake server+;;;+;;; A raw-socket LLM server that streams SSE with `Transfer-Encoding: chunked`, for tests that must see when+;;; the client drops the connection. Apache HttpClient drains the rest of a chunked body when the body+;;; closes: https://github.com/dakrone/clj-http/issues/627+;;; ──────────────────────────────────────────────────────────────────++(defn- read-head!+ "Read an HTTP request head from `in`, up to the blank line, and return its lines. Nil at EOF."+ [^InputStream in]+ (let [sb (StringBuilder.)]+ (loop []+ (let [b (.read in)]+ (cond+ (neg? b) nil+ (str/ends-with? (.append sb (char b)) "\r\n\r\n") (str/split-lines (str sb))+ :else (recur))))))++(defn- header-value+ "The value of the header `header-name` (lower case) in the head `lines`, or nil."+ [lines header-name]+ (some (fn [line]+ (let [[k v] (str/split line #":\s*" 2)]+ (when (= header-name (u/lower-case-en k))+ v)))+ lines))++(defn- read-request!+ "Read one request from `in`. Return its `:method`, its `:path` and `:max-tokens?`, true when the JSON body+ has a `max_tokens` key. Nil at EOF."+ [^InputStream in]+ (when-let [[request-line & header-lines] (read-head! in)]+ (let [[method path] (str/split request-line #" ")+ length (parse-long (or (header-value header-lines "content-length") "0"))+ body (String. (.readNBytes in (int length)) "UTF-8")]+ {:method method+ :path path+ :max-tokens? (str/includes? body "\"max_tokens\"")})))++(defn- write-ascii!+ [^OutputStream out ^String s]+ (.write out (.getBytes s "US-ASCII")))++(defn- write-chunk!+ "Write `data` to `out` as one HTTP chunk, and flush it. Write nothing for empty `data`."+ [^OutputStream out ^bytes data]+ (when (pos? (alength data))+ (write-ascii! out (str (Integer/toHexString (alength data)) "\r\n"))+ (.write out data)+ (write-ascii! out "\r\n")+ (.flush out)))++(defn- plain-encoder+ "An encoder that sends text as UTF-8 bytes."+ []+ {:encode (fn [^String s] (.getBytes s "UTF-8"))+ :finish (fn [] (byte-array 0))})++(defn- gzip-encoder+ "An encoder that sends text as one gzip stream. It flushes after each piece, so the client can read it at once."+ []+ (let [buf (ByteArrayOutputStream.)+ gz (GZIPOutputStream. buf true)+ drain! (fn [] (let [data (.toByteArray buf)] (.reset buf) data))]+ {:encode (fn [^String s] (.write gz (.getBytes s "UTF-8")) (.flush gz) (drain!))+ :finish (fn [] (.finish gz) (drain!))}))++(defn- response-head+ [gzip?]+ (str "HTTP/1.1 200 OK\r\n"+ "Content-Type: text/event-stream\r\n"+ "Transfer-Encoding: chunked\r\n"+ "Connection: close\r\n"+ (when gzip? "Content-Encoding: gzip\r\n")+ "\r\n"))++(defn- default-event+ "The `i`th event of a stream: an OpenAI Chat Completions text delta."+ [i]+ {:id "chatcmpl-fake"+ :model "fake-model"+ :choices [{:index 0+ :delta {:role "assistant" :content (str "chunk-" i " ")}+ :finish_reason nil}]})++(defn- write-events!+ "Write `n` SSE events, one every `interval-ms`. Call `on-sent` with the count after each event."+ [write! n {:keys [interval-ms event-fn]} on-sent]+ (dotimes [i n]+ (write! (str "data: " (json/encode (event-fn i)) "\n\n"))+ (on-sent (inc i))+ (Thread/sleep (long interval-ms))))++(defn- write-after!+ "Write what comes after the events: `[DONE]` for `:done`, nothing for `:eof`, and for `:keep-alive` `[DONE]`+ and then 20 s of `: ping` comments."+ [write! after]+ (case after+ :done (write! "data: [DONE]\n\n")+ :eof nil+ :keep-alive (do (write! "data: [DONE]\n\n")+ (dotimes [_ 2000]+ (write! ": ping\n\n")+ (Thread/sleep 10)))))++(defn- stream-response!+ "Write a chunked SSE response of `n` events to `out`. Throws an `IOException` when the client is gone."+ [^OutputStream out n {:keys [gzip? after] :as opts} on-sent]+ (let [{:keys [encode finish]} (if gzip? (gzip-encoder) (plain-encoder))+ write! #(write-chunk! out (encode %))]+ (write-ascii! out (response-head gzip?))+ (.flush out)+ (write-events! write! n opts on-sent)+ (write-after! write! after)+ (write-chunk! out (finish))+ (write-ascii! out "0\r\n\r\n")+ (.flush out)))++(defn- event-count+ "How many events the fake sends for `request`: `:short-total` for a capped (`max_tokens`) call, else `:total`."+ [{:keys [max-tokens?]} {:keys [total short-total]}]+ (if (and short-total max-tokens?) short-total total))++(defn- add-connection!+ "Add the record of a new connection for `request` to `connections`, and return its index."+ [connections request]+ (dec (count (swap! connections conj (assoc request :sent 0 :disconnected-at-ms nil)))))++(defn- serve-post!+ "Stream the response to one `POST`. The first `POST` also reports to the server's `:sent` and `:disconnected`."+ [^OutputStream out request opts {:keys [connections sent disconnected first-post]}]+ (let [idx (add-connection! connections request)+ first? (compare-and-set! first-post false true)+ count! (fn [n]+ (swap! connections assoc-in [idx :sent] n)+ (when first? (reset! sent n)))]+ (try+ (stream-response! out (event-count request opts) opts count!)+ (catch IOException _+ (swap! connections assoc-in [idx :disconnected-at-ms] (System/currentTimeMillis))+ (when first?+ (deliver disconnected (get-in @connections [idx :sent])))))))++(defn- serve-not-found!+ [^OutputStream out request {:keys [connections]}]+ (add-connection! connections request)+ (write-ascii! out "HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")+ (.flush out))++(defn- serve-connection!+ "Serve the one request of `socket`: a `POST` gets the stream, any other method a 404."+ [^Socket socket opts state]+ (with-open [socket socket]+ (let [out (.getOutputStream socket)+ request (read-request! (.getInputStream socket))]+ (cond+ (nil? request) nil+ (= "POST" (:method request)) (serve-post! out request opts state)+ :else (serve-not-found! out request state)))))++(defn- accept-loop!+ "Accept connections on `server` until it closes, and serve each one on its own thread."+ [^ServerSocket server opts {:keys [sockets] :as state}]+ (future+ (try+ (loop []+ (let [socket (.accept server)]+ (swap! sockets conj socket)+ (future+ (try+ (serve-connection! socket opts state)+ (catch Exception e+ (log/debug e "Fake SSE server: a connection failed"))))+ (recur)))+ (catch SocketException _+ nil))))++(defn start-chunked-sse-server!+ "Start a fake LLM server on localhost that answers each `POST` with a chunked SSE stream.++ Options (all optional):++ :total - events per stream (default 10)+ :interval-ms - pause after each event (default 10)+ :event-fn - fn of the event index that returns the event map (default: an OpenAI Chat Completions+ text delta)+ :after - what follows the events: `:done` (`data: [DONE]`, the default), `:eof` (nothing) or+ `:keep-alive` (`[DONE]`, then 20 s of `: ping` comments). The chunked terminator follows.+ :gzip? - send the body with `Content-Encoding: gzip`+ :port - the port to listen on (default 0, a free port)+ :short-total - events for a `POST` whose JSON body has `max_tokens`, such as a title call++ Any other method gets a 404. Returns:++ :url - the base URL, e.g. `http://127.0.0.1:52011`+ :sent - atom: the events sent on the first `POST`+ :disconnected - promise: the `:sent` count when a write to the first `POST` failed (the client left)+ :connections - atom: one map per request, `{:method :path :max-tokens? :sent :disconnected-at-ms}`+ :stop! - fn that closes the server and every open connection"+ ([]+ (start-chunked-sse-server! {}))+ ([opts]+ (let [opts (merge {:total 10 :interval-ms 10 :event-fn default-event :after :done :port 0} opts)+ server (ServerSocket. (int (:port opts)) 50 (InetAddress/getLoopbackAddress))+ state {:connections (atom [])+ :sent (atom 0)+ :disconnected (promise)+ :first-post (atom false)+ :sockets (atom [])}]+ (accept-loop! server opts state)+ {:url (str "http://127.0.0.1:" (.getLocalPort server))+ :sent (:sent state)+ :disconnected (:disconnected state)+ :connections (:connections state)+ :stop! (fn []+ (.close server)+ (run! #(.close ^Socket %) @(:sockets state)))})))