Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Drop the upstream LLM connection when a Metabot stream closes (#85023)

Paolo Rascunà67b958b8857f
diff --git a/src/metabase/metabot/self/adapter.clj b/src/metabase/metabot/self/adapter.clj
index a4702755c73..56d0eacea63 100644
--- a/src/metabase/metabot/self/adapter.clj
+++ b/src/metabase/metabot/self/adapter.clj
@@ -406,7 +406,9 @@
depends on the connection rather than only on the response. Applies to both
phases below.
:read-stream - turns the response body into a reducible of the provider's events, for a stream
- that is not SSE. Defaults to [[core/sse-reducible]].
+ that is not SSE. Defaults to [[core/sse-reducible]]. When the request has its own
+ client, a close of the body it gets drops the connection and does not read
+ the rest of the body (see [[core/abort-on-close]]).
:wrap-stream - applied to the reducible before error translation, for an adapter with its own
translation to do first.
:on-request-error - replaces the default [[rethrow!]] catch, for an adapter that retries. Only
@@ -445,7 +447,8 @@
;; encoded up front, since a provider may sign over the body
:body (json/encode body)}
request-options)
- :body
+ ;; on this thread, right after the request, so it sees the connection manager clj-http saw
+ (core/abort-on-close request-options)
read-stream
(debug/capture-stream {:provider slug
:model model
diff --git a/src/metabase/metabot/self/core.clj b/src/metabase/metabot/self/core.clj
index 6beb76fa1d8..4d84b2af858 100644
--- a/src/metabase/metabot/self/core.clj
+++ b/src/metabase/metabot/self/core.clj
@@ -1,6 +1,7 @@
(ns metabase.metabot.self.core
(:require
[clj-http.client :as http]
+ [clj-http.conn-mgr :as conn]
[clojure.java.io :as io]
[clojure.string :as str]
[clojure.walk :as walk]
@@ -21,7 +22,7 @@
[metabase.util.malli.schema :as ms]
[metabase.util.o11y :refer [with-span]])
(:import
- (java.io BufferedReader Closeable InputStream)
+ (java.io BufferedReader Closeable FilterInputStream InputStream IOException)
(java.util.concurrent Callable Executors ExecutorService)))
(set! *warn-on-reflection* true)
@@ -357,6 +358,54 @@
(close [_]
(.close input))))
+(defn- close-released-body!
+ "Close a streamed `body` whose client is already closed. Ignore the error that this close is known to throw,
+ and log it at debug level."
+ [^InputStream body]
+ ;; The body close releases the connection to a manager that is shut down. Apache HttpClient then throws
+ ;; `IllegalStateException: Connection not obtained from this manager`, or an `IOException` from the closed
+ ;; socket. See https://github.com/dakrone/clj-http/issues/627
+ (try
+ (.close body)
+ (catch IllegalStateException e
+ (log/debug e "Ignored the error from the body close after the client close"))
+ (catch IOException e
+ (log/debug e "Ignored the error from the body close after the client close"))))
+
+(defn- own-client?
+ "True when clj-http made the client of a request for that request only.
+
+ A bound or passed connection manager, or a passed client, can serve other requests."
+ [request-options]
+ (and (nil? conn/*connection-manager*)
+ (nil? (:connection-manager request-options))
+ (nil? (:http-client request-options))))
+
+(defn abort-on-close
+ "Return the body of a streamed clj-http `response`, changed so that a close drops the connection.
+
+ A close of the returned body closes the request's own `:http-client` first, so the provider sees a disconnect.
+ Then it closes the body, and it does not read the rest of the body. Over TLS 1.3 the client close can wait for
+ the next byte from the provider, up to the socket timeout.
+
+ Returns `:body` unchanged when it is not an `InputStream`, when the response has no `Closeable` `:http-client`,
+ or when `request-options` (the clj-http options of the request) share a client or a connection manager with
+ other requests. Call it on the thread that made the request, so it sees the same bound connection manager.
+ This is the workaround from https://github.com/dakrone/clj-http/issues/627."
+ [{:keys [body http-client]} request-options]
+ (if (and (instance? InputStream body)
+ (instance? Closeable http-client)
+ (own-client? request-options))
+ ;; Without this wrapper, a close of a chunked body reads the rest of the body. It returns only when the provider
+ ;; finishes, and the provider continues to generate until then.
+ (proxy [FilterInputStream] [body]
+ (close []
+ (try
+ (.close ^Closeable http-client)
+ (finally
+ (close-released-body! body)))))
+ body))
+
;;; AISDK5
(def finish-reasons
diff --git a/test/metabase/metabot/api_test.clj b/test/metabase/metabot/api_test.clj
index e5a508271c6..9120706bcdf 100644
--- a/test/metabase/metabot/api_test.clj
+++ b/test/metabase/metabot/api_test.clj
@@ -463,6 +463,112 @@
^String [data]
(str "data: " (json/encode data) "\n\n"))
+(defn- fake-openrouter-handler
+ "A fake OpenRouter (Chat Completions) server: an async ring handler that streams SSE text deltas, one every
+ 10 ms, while `cnt` is positive. It decrements `cnt` after each delta, and stops when its consumer disconnects.
+
+ When it stops with `cnt` still positive, it delivers the remaining `cnt` to the optional `:stopped-early` promise."
+ [cnt & {:keys [stopped-early]}]
+ (let [chat-id (str "chatcmpl-" (random-uuid))]
+ (fn [req respond _raise]
+ (respond
+ (compojure.response/render
+ (sr/streaming-response {:content-type "text/event-stream; charset=utf-8"} [os llm-canceled-chan]
+ (try
+ (let [write! (fn [^String s]
+ (.write os (.getBytes s "UTF-8"))
+ (.flush os))
+ canceled? #(some? (a/poll! llm-canceled-chan))]
+ ;; First chunk: role assignment (empty content to establish assistant role)
+ (write! (sse-event {:id chat-id
+ :model "anthropic/claude-haiku-4-5"
+ :choices [{:index 0
+ :delta {:role "assistant" :content ""}
+ :finish_reason nil}]}))
+ ;; Stream text content chunks slowly, stop early if the consumer disconnects
+ (loop []
+ (when (and (pos? @cnt) (not (canceled?)))
+ (write! (sse-event {:id chat-id
+ :model "anthropic/claude-haiku-4-5"
+ :choices [{:index 0
+ ;; pad past Jetty's 8 KB buffer so .flush reaches the client mid-stream
+ :delta {:content (str "chunk-" @cnt " " (apply str (repeat 512 \x)))}
+ :finish_reason nil}]}))
+ (swap! cnt dec)
+ (Thread/sleep 10)
+ (recur)))
+ (when-not (canceled?)
+ ;; Finish reason
+ (write! (sse-event {:id chat-id
+ :model "anthropic/claude-haiku-4-5"
+ :choices [{:index 0
+ :delta {}
+ :finish_reason "stop"}]}))
+ ;; Usage (separate final chunk, as OpenRouter does)
+ (write! (sse-event {:id chat-id
+ :model "anthropic/claude-haiku-4-5"
+ :choices []
+ :usage {:prompt_tokens 10
+ :completion_tokens 50}}))
+ (write! "data: [DONE]\n\n")))
+ (catch Exception _e nil)
+ ;; A write after the disconnect can throw before `canceled?` turns true. So the `finally` reports the
+ ;; early stop, which covers both paths.
+ (finally
+ (when (and stopped-early (pos? @cnt))
+ (deliver stopped-early @cnt)))))
+ req)))))
+
+(defn- do-with-closed-agent-stream!
+ "Start a native-agent turn against the fake LLM at `llm-url`, read one byte of the response, and close the
+ client. Wait up to 3 s for `finalize-assistant-turn!`, then call `f` with `:stored-parts`, `:stored-kwargs`
+ (the finalize arguments, nil if it did not run) and the `:conversation-id`."
+ [llm-url f]
+ (let [stored-parts (atom nil)
+ stored-kwargs (atom nil)]
+ (mt/test-helpers-set-global-values!
+ (search.tu/with-index-disabled
+ ;; the fake LLM server is on localhost, which the network policy refuses on a hosted instance
+ (mt/with-temp-env-var-value! [mb-llm-allowed-networks "allow-all"]
+ (mt/with-temporary-setting-values [llm.settings/llm-providers [(llm.tu/connection "openrouter" {:base-url llm-url})]
+ metabot.settings/llm-metabot-provider test-provider]
+ (let [real-llm-request self.core/request]
+ (with-redefs [scope/resolve-user-permissions (constantly scope/all-yes-permissions)
+ conversation-title/ensure-title! (constantly {:status :missing})
+ ;; The fake LLM server gzips whenever the caller accepts it, and clj-http
+ ;; wraps the body in a GZIPInputStream. Closing mid-stream causes ZLIB errors.
+ self.core/request (fn [auth req]
+ (real-llm-request auth (assoc req :decompress-body false)))
+ metabot.context/create-context (fn [ctx & _] ctx)
+ metabot.persistence/finalize-assistant-turn! (fn [_pk parts & kwargs]
+ (reset! stored-parts parts)
+ (reset! stored-kwargs (apply hash-map kwargs)))
+ sr/async-cancellation-poll-interval-ms 5]
+ (mt/with-model-cleanup [:model/MetabotMessage
+ [:model/MetabotConversation :created_at]]
+ (let [conversation-id (str (random-uuid))
+ response (mt/user-real-request-full-response
+ :rasta :post 202 "metabot/agent-streaming"
+ {:request-options {:as :stream
+ :decompress-body false}}
+ {:message "Test closure"
+ :context {}
+ :conversation_id conversation-id
+ :state {}})]
+ (.read ^java.io.InputStream (:body response)) ;; start the handler
+ ;; Close the underlying client, not the body stream: closing the body would
+ ;; make clj-http drain the (now chunked) response to completion, which looks
+ ;; like a normal finish rather than a disconnect. Closing the client aborts
+ ;; the connection, which is what the server's cancel loop detects.
+ (.close ^java.io.Closeable (:http-client response))
+ (u/poll {:thunk #(deref stored-parts)
+ :done? some?
+ :interval-ms 10
+ :timeout-ms 3000})
+ (f {:stored-parts @stored-parts
+ :stored-kwargs @stored-kwargs
+ :conversation-id conversation-id})))))))))))
+
(deftest closing-connection-native-agent-test
(testing "When the client closes a native-agent streaming connection, the
pipeline tears down and the partial turn is persisted as aborted."
@@ -476,107 +582,41 @@
;; the agent loop unwinds, and `finalize-assistant-turn!` is called from the
;; `finally` with `:finished? false`, UPDATEing the placeholder row inserted
;; by `start-turn!`.
- (let [total-chunks 30
+ (let [cnt (atom 30)]
+ (server.tu/with-test-server [llm-url (fake-openrouter-handler cnt)]
+ (testing "Closing stream body tears down the pipeline and persists the aborted turn"
+ (do-with-closed-agent-stream!
+ llm-url
+ (fn [{:keys [stored-parts stored-kwargs conversation-id]}]
+ (is (some? stored-parts)
+ "finalize-assistant-turn! was called even though the client disconnected")
+ (is (false? (:finished? stored-kwargs))
+ "the finalized turn is marked :finished? false — the cancel was detected")
+ (is (= 2 (count (t2/select :model/MetabotMessage
+ :conversation_id conversation-id)))
+ "start-turn! inserted exactly user + placeholder; no extra row from finalize"))))))))
+
+(deftest closing-connection-stops-upstream-llm-test
+ (testing "When the client closes a native-agent streaming connection during a long LLM stream, Metabase drops
+ its connection to the LLM, so the LLM stops sending and the turn finalizes at once"
+ ;; Before the workaround for https://github.com/dakrone/clj-http/issues/627, a close of the LLM response body
+ ;; read the rest of the chunked stream. The turn then finalized only when the LLM finished (about 20 s here).
+ (let [total-chunks 2000
cnt (atom total-chunks)
- stored-parts (atom nil)
- stored-kwargs (atom nil)
- chat-id (str "chatcmpl-" (random-uuid))
- ;; Fake OpenRouter API: streams Chat Completions SSE text deltas.
- llm-handler
- (fn [req respond _raise]
- (respond
- (compojure.response/render
- (sr/streaming-response {:content-type "text/event-stream; charset=utf-8"} [os llm-canceled-chan]
- (try
- (let [write! (fn [^String s]
- (.write os (.getBytes s "UTF-8"))
- (.flush os))
- canceled? #(some? (a/poll! llm-canceled-chan))]
- ;; First chunk: role assignment (empty content to establish assistant role)
- (write! (sse-event {:id chat-id
- :model "anthropic/claude-haiku-4-5"
- :choices [{:index 0
- :delta {:role "assistant" :content ""}
- :finish_reason nil}]}))
- ;; Stream text content chunks slowly, stop early if the consumer disconnects
- (loop []
- (when (and (pos? @cnt) (not (canceled?)))
- (write! (sse-event {:id chat-id
- :model "anthropic/claude-haiku-4-5"
- :choices [{:index 0
- ;; pad past Jetty's 8 KB buffer so .flush reaches the client mid-stream
- :delta {:content (str "chunk-" @cnt " " (apply str (repeat 512 \x)))}
- :finish_reason nil}]}))
- (swap! cnt dec)
- (Thread/sleep 10)
- (recur)))
- (when-not (canceled?)
- ;; Finish reason
- (write! (sse-event {:id chat-id
- :model "anthropic/claude-haiku-4-5"
- :choices [{:index 0
- :delta {}
- :finish_reason "stop"}]}))
- ;; Usage (separate final chunk, as OpenRouter does)
- (write! (sse-event {:id chat-id
- :model "anthropic/claude-haiku-4-5"
- :choices []
- :usage {:prompt_tokens 10
- :completion_tokens 50}}))
- (write! "data: [DONE]\n\n")))
- (catch Exception _e nil)))
- req)))]
- (server.tu/with-test-server [llm-url llm-handler]
- (mt/test-helpers-set-global-values!
- (search.tu/with-index-disabled
- ;; the fake LLM server is on localhost, which the network policy refuses on a hosted instance
- (mt/with-temp-env-var-value! [mb-llm-allowed-networks "allow-all"]
- (mt/with-temporary-setting-values [llm.settings/llm-providers [(llm.tu/connection "openrouter" {:base-url llm-url})]
- metabot.settings/llm-metabot-provider test-provider]
- (let [real-llm-request self.core/request]
- (with-redefs [scope/resolve-user-permissions (constantly scope/all-yes-permissions)
- conversation-title/ensure-title! (constantly {:status :missing})
- ;; The fake LLM server gzips whenever the caller accepts it, and clj-http
- ;; wraps the body in a GZIPInputStream. Closing mid-stream causes ZLIB errors.
- self.core/request (fn [auth req]
- (real-llm-request auth (assoc req :decompress-body false)))
- metabot.context/create-context (fn [ctx & _] ctx)
- metabot.persistence/finalize-assistant-turn! (fn [_pk parts & kwargs]
- (reset! stored-parts parts)
- (reset! stored-kwargs (apply hash-map kwargs)))
- sr/async-cancellation-poll-interval-ms 5]
- (testing "Closing stream body tears down the pipeline and persists the aborted turn"
- (reset! cnt total-chunks)
- (reset! stored-parts nil)
- (reset! stored-kwargs nil)
- (mt/with-model-cleanup [:model/MetabotMessage
- [:model/MetabotConversation :created_at]]
- (let [conversation-id (str (random-uuid))
- response (mt/user-real-request-full-response
- :rasta :post 202 "metabot/agent-streaming"
- {:request-options {:as :stream
- :decompress-body false}}
- {:message "Test closure"
- :context {}
- :conversation_id conversation-id
- :state {}})]
- (.read ^java.io.InputStream (:body response)) ;; start the handler
- ;; Close the underlying client, not the body stream: closing the body would
- ;; make clj-http drain the (now chunked) response to completion, which looks
- ;; like a normal finish rather than a disconnect. Closing the client aborts
- ;; the connection, which is what the server's cancel loop detects.
- (.close ^java.io.Closeable (:http-client response))
- (u/poll {:thunk #(deref stored-parts)
- :done? some?
- :interval-ms 10
- :timeout-ms 3000})
- (is (some? @stored-parts)
- "finalize-assistant-turn! was called even though the client disconnected")
- (is (false? (:finished? @stored-kwargs))
- "the finalized turn is marked :finished? false — the cancel was detected")
- (is (= 2 (count (t2/select :model/MetabotMessage
- :conversation_id conversation-id)))
- "start-turn! inserted exactly user + placeholder; no extra row from finalize"))))))))))))))
+ stopped-early (promise)]
+ (server.tu/with-test-server [llm-url (fake-openrouter-handler cnt :stopped-early stopped-early)]
+ (do-with-closed-agent-stream!
+ llm-url
+ (fn [{:keys [stored-parts stored-kwargs]}]
+ (is (some? stored-parts)
+ "finalize-assistant-turn! ran within 3 s of the disconnect")
+ (is (false? (:finished? stored-kwargs))
+ "the finalized turn is marked :finished? false")
+ (let [remaining (deref stopped-early 2000 nil)]
+ (is (some? remaining)
+ "the fake LLM saw the disconnect and stopped before it sent all its chunks")
+ (is (< (- total-chunks (or remaining 0)) 300)
+ "the fake LLM saw the disconnect long before it sent all its chunks"))))))))
(deftest turn-longer-than-async-response-timeout-finishes-test
(testing "A turn that outlasts MB_JETTY_ASYNC_RESPONSE_TIMEOUT streams its finish event and finalizes as finished"
diff --git a/test/metabase/metabot/self/adapter_test.clj b/test/metabase/metabot/self/adapter_test.clj
index 94c666684d1..2e94b4f34b2 100644
--- a/test/metabase/metabot/self/adapter_test.clj
+++ b/test/metabase/metabot/self/adapter_test.clj
@@ -1,6 +1,7 @@
(ns metabase.metabot.self.adapter-test
(:require
[clj-http.client :as http]
+ [clj-http.conn-mgr :as conn]
[clojure.edn :as edn]
[clojure.string :as str]
[clojure.test :refer [deftest is testing]]
@@ -23,9 +24,12 @@
[metabase.metabot.self.vllm :as vllm]
[metabase.metabot.self.xai :as xai]
[metabase.metabot.self.zai :as zai]
+ [metabase.metabot.test-util :as mut]
[metabase.premium-features.core :as premium-features]
[metabase.test :as mt]
- [metabase.util.malli.registry :as mr]))
+ [metabase.util.malli.registry :as mr])
+ (:import
+ (java.io ByteArrayInputStream Closeable InputStream IOException)))
(set! *warn-on-reflection* true)
@@ -379,3 +383,256 @@
(is (= ["From name"]
(names zai/list-models
{:data [{:id "glm-5.2" :name "From name" :display_name "From display_name"}]}))))))
+
+;;; ──────────────────────────────────────────────────────────────────
+;;; Upstream connection on close
+;;;
+;;; A close of a clj-http `:as :stream` body reads the rest of a chunked body before it returns, so the
+;;; provider continues to generate: https://github.com/dakrone/clj-http/issues/627. These tests check that a
+;;; stream from [[adapter/stream!]] drops the connection when its body closes.
+;;; ──────────────────────────────────────────────────────────────────
+
+(defn- timed-reduce
+ "Call `thunk`. Return its `:result`, the exception it threw as `:thrown`, and the `:elapsed-ms`."
+ [thunk]
+ (let [start (System/nanoTime)
+ [result thrown] (try
+ [(thunk) nil]
+ (catch Throwable e
+ [nil e]))]
+ {:result result
+ :thrown thrown
+ :elapsed-ms (quot (- (System/nanoTime) start) 1000000)}))
+
+(defn- with-chunked-server!
+ "Call `(f srv)` with a fresh [[mut/start-chunked-sse-server!]] made from `opts`, with localhost allowed. Stop the
+ server after."
+ [opts f]
+ (let [srv (mut/start-chunked-sse-server! opts)]
+ (try
+ (mt/with-temp-env-var-value! [mb-llm-allowed-networks "allow-all"]
+ (f srv))
+ (finally
+ ((:stop! srv))))))
+
+(defn- server-request-opts
+ [srv]
+ {:model "fake-model"
+ :input [{:role :user :content "hi"}]
+ :credentials {:api-key "sk-or-test" :base-url (:url srv)}})
+
+(defn- openrouter-stream
+ [srv]
+ (openrouter/openrouter-raw (server-request-opts srv)))
+
+(defn- take-events
+ "A reducing fn that collects events, and stops after `n`."
+ [n]
+ (fn [acc event]
+ (let [acc (conj acc event)]
+ (if (= n (count acc))
+ (reduced acc)
+ acc))))
+
+(defn- disconnected-count
+ "The events the server sent before it saw the disconnect, or nil if it saw none within 2 s."
+ [srv]
+ (deref (:disconnected srv) 2000 nil))
+
+(defn- dropped-early?
+ "True when the server saw the disconnect after fewer than `n` events."
+ [srv n]
+ (< (or (disconnected-count srv) ##Inf) n))
+
+(defn- cause-chain
+ [e]
+ (take-while some? (iterate ex-cause e)))
+
+(deftest stream-close-drops-upstream-connection-test
+ (testing "a reduction that stops early drops the upstream connection, and does not read the rest of the body"
+ (with-chunked-server! {:total 2000 :interval-ms 10}
+ (fn [srv]
+ (let [{:keys [result thrown elapsed-ms]} (timed-reduce #(reduce (take-events 3) [] (openrouter-stream srv)))]
+ (is (nil? thrown))
+ (is (= 3 (count result)))
+ (is (< elapsed-ms 2000))
+ (is (dropped-early? srv 100)))))))
+
+(deftest stream-done-before-eof-drops-upstream-connection-test
+ (testing "a stream that ends at `[DONE]` drops the connection, also when the server sends more bytes after it"
+ (with-chunked-server! {:total 3 :after :keep-alive}
+ (fn [srv]
+ (let [{:keys [result thrown elapsed-ms]} (timed-reduce #(reduce conj [] (openrouter-stream srv)))]
+ (is (nil? thrown))
+ (is (= 3 (count result)))
+ (is (< elapsed-ms 2000))
+ (is (some? (disconnected-count srv))))))))
+
+(deftest stream-gzip-close-drops-upstream-connection-test
+ (testing "a gzip body, which clj-http decompresses, also drops the connection when it closes"
+ (with-chunked-server! {:total 2000 :interval-ms 10 :gzip? true}
+ (fn [srv]
+ (let [{:keys [result thrown elapsed-ms]} (timed-reduce #(reduce (take-events 3) [] (openrouter-stream srv)))]
+ (is (nil? thrown))
+ (is (= 3 (count result)))
+ (is (< elapsed-ms 2000))
+ (is (dropped-early? srv 100)))))))
+
+(defn- read-one-buffer
+ "A `:read-stream` like Bedrock's `runtime-events`: its reduction opens `body` with `with-open`, reads one buffer
+ and returns `init`."
+ [^InputStream body]
+ (reify clojure.lang.IReduceInit
+ (reduce [_ _rf init]
+ (with-open [in body]
+ (.read in (byte-array 8192))
+ init))))
+
+(deftest stream-custom-read-stream-drops-upstream-connection-test
+ (testing "a `:read-stream` that is not SSE, and closes the body itself, also drops the connection"
+ (with-chunked-server! {:total 2000 :interval-ms 10}
+ (fn [srv]
+ (let [{:keys [thrown elapsed-ms]} (timed-reduce
+ #(reduce conj []
+ (adapter/stream! @#'openrouter/provider
+ (server-request-opts srv)
+ {:path "/v1/chat/completions"
+ :body {}
+ :read-stream read-one-buffer})))]
+ (is (nil? thrown))
+ (is (< elapsed-ms 2000))
+ (is (dropped-early? srv 100)))))))
+
+(defn- recording-body
+ "An `InputStream` over `text` that adds `:body` to `log` when it closes, and then throws `close-ex` if given."
+ ^InputStream [log ^String text close-ex]
+ (proxy [ByteArrayInputStream] [(.getBytes text "UTF-8")]
+ (close []
+ (swap! log conj :body)
+ (when close-ex
+ (throw close-ex)))))
+
+(defn- recording-client
+ "A `Closeable` that adds `:client` to `log` when it closes."
+ [log]
+ (reify Closeable
+ (close [_]
+ (swap! log conj :client))))
+
+(def ^:private stub-sse
+ "data: {\"n\":1}\n\ndata: {\"n\":2}\n\ndata: [DONE]\n\n")
+
+(def ^:private stub-request-opts
+ {:model "fake-model"
+ :input [{:role :user :content "hi"}]
+ :credentials {:api-key "sk-or-test" :base-url "https://openrouter.ai/api"}})
+
+(defn- stub-stream!
+ "Reduce [[adapter/stream!]] to the end, with `http/request` stubbed to return `response`. `stream-opts` merge
+ into the minimal stream opts."
+ [response stream-opts]
+ (with-redefs [http/request (fn [_] (assoc response :status 200))]
+ (timed-reduce #(reduce conj []
+ (adapter/stream! @#'openrouter/provider
+ stub-request-opts
+ (merge {:path "/v1/chat/completions" :body {}} stream-opts))))))
+
+(defn- recorded-close-stream!
+ "[[stub-stream!]] over a recording body that throws `close-ex` on close, with a recording client. Adds the
+ close log to the result as `:closed`."
+ [close-ex stream-opts]
+ (let [log (atom [])]
+ (assoc (stub-stream! {:body (recording-body log stub-sse close-ex)
+ :http-client (recording-client log)}
+ stream-opts)
+ :closed @log)))
+
+(deftest stream-close-order-test
+ (testing "the client closes before the body, so the body close does not read the rest of the stream"
+ (let [{:keys [result thrown closed]} (recorded-close-stream! nil nil)]
+ (is (nil? thrown))
+ (is (= [{:n 1} {:n 2}] result))
+ (is (= [:client :body] closed)))))
+
+(deftest stream-close-ignores-released-connection-error-test
+ (testing "the error that the body close throws after the client closes is ignored"
+ (testing "IllegalStateException"
+ (let [{:keys [result thrown]} (recorded-close-stream!
+ (IllegalStateException. "Connection not obtained from this manager")
+ nil)]
+ (is (nil? thrown))
+ (is (= [{:n 1} {:n 2}] result))))
+ (testing "IOException"
+ (let [{:keys [result thrown]} (recorded-close-stream! (IOException. "Socket closed") nil)]
+ (is (nil? thrown))
+ (is (= [{:n 1} {:n 2}] result))))))
+
+(deftest stream-close-keeps-other-close-errors-test
+ (testing "any other error from the body close reaches the caller as a provider error"
+ (let [boom (RuntimeException. "boom")
+ {:keys [thrown]} (recorded-close-stream! boom nil)]
+ (is (= :provider-request-failed (:error-code (ex-data thrown))))
+ (is (identical? boom (ex-cause thrown))))))
+
+(deftest stream-rf-exception-survives-close-test
+ (testing "an exception from `rf` reaches the caller; the close error does not replace it"
+ (with-chunked-server! {:total 300 :interval-ms 10}
+ (fn [srv]
+ (let [rf (fn [acc event]
+ (when (= 1 (count acc))
+ (throw (ex-info "boom" {::boom true})))
+ (conj acc event))
+ {:keys [thrown elapsed-ms]} (timed-reduce #(reduce rf [] (openrouter-stream srv)))
+ chain (cause-chain thrown)]
+ (is (some (comp ::boom ex-data) chain))
+ (is (not-any? #(instance? IllegalStateException %) chain))
+ (is (< elapsed-ms 2000))
+ (is (dropped-early? srv 100)))))))
+
+(deftest stream-normal-finish-raises-no-error-test
+ (testing "a stream that reads to its end gives every event, no exception and no warning"
+ (doseq [after [:done :eof]]
+ (testing after
+ (with-chunked-server! {:total 5 :after after}
+ (fn [srv]
+ (mt/with-log-messages-for-level [messages [metabase.metabot.self :warn]]
+ (let [{:keys [result thrown]} (timed-reduce #(reduce conj [] (openrouter-stream srv)))]
+ (is (nil? thrown))
+ (is (= 5 (count result)))
+ (is (empty? (messages)))))))))))
+
+(deftest stream-close-error-logged-at-debug-test
+ (testing "a stream that ends at `[DONE]` logs the ignored body-close error at debug level"
+ (with-chunked-server! {:total 5 :after :done}
+ (fn [srv]
+ (mt/with-log-messages-for-level [messages [metabase.metabot.self.core :debug]]
+ (let [{:keys [thrown]} (timed-reduce #(reduce conj [] (openrouter-stream srv)))]
+ (is (nil? thrown))
+ (is (=? [{:level :debug
+ :message "Ignored the error from the body close after the client close"}]
+ (messages)))))))))
+
+(deftest stream-keeps-shared-client-test
+ (testing "a client that can serve other requests stays open; only the body closes"
+ (let [cm (conn/make-reusable-conn-manager {})]
+ (try
+ (testing "a bound connection manager"
+ (binding [conn/*connection-manager* cm]
+ (is (= [:body] (:closed (recorded-close-stream! nil nil))))))
+ (testing "a connection manager in the request options"
+ (is (= [:body] (:closed (recorded-close-stream! nil {:request-options {:connection-manager cm}})))))
+ (testing "an http client in the request options"
+ (is (= [:body] (:closed (recorded-close-stream! nil {:request-options {:http-client (recording-client (atom []))}})))))
+ (finally
+ (conn/shutdown-manager cm))))))
+
+(deftest stream-passes-through-stub-bodies-test
+ (testing "a response the helper cannot abort reaches `read-stream` as the identical body"
+ (doseq [[label response] [["a map body" {:body {:a 1}}]
+ ["a stream with no client" {:body (ByteArrayInputStream. (byte-array 0))}]
+ ["a client that is not Closeable" {:body (ByteArrayInputStream. (byte-array 0))
+ :http-client (Object.)}]]]
+ (testing label
+ (let [seen (atom nil)]
+ (stub-stream! response {:read-stream (fn [body] (reset! seen body) [])})
+ (is (identical? (:body response) @seen)))))))
diff --git a/test/metabase/metabot/self_test.clj b/test/metabase/metabot/self_test.clj
index ba78b10d171..3c3b67f4a2e 100644
--- a/test/metabase/metabot/self_test.clj
+++ b/test/metabase/metabot/self_test.clj
@@ -39,8 +39,7 @@
[metabase.util.json :as json]
[metabase.util.log.capture :as log.capture]
[metabase.util.malli :as mu]
- [metabase.util.malli.registry :as mr]
- [ring.adapter.jetty :as jetty]))
+ [metabase.util.malli.registry :as mr]))
(set! *warn-on-reflection* true)
@@ -428,48 +427,6 @@
(self.core/sse-reducible istream))]
(is (= ["msg-0" "msg-1" "msg-2"] result)))))
-(deftest sse-reducible-stops-response-test
- (let [cnt (atom 30)
- handler (fn [_req]
- (let [out (java.io.PipedOutputStream.)
- in (java.io.PipedInputStream. out 65536)]
- (future
- (try
- (dotimes [i @cnt]
- (.write out (.getBytes (test-util/make-sse [(str "msg-" i)])))
- (.flush out)
- (swap! cnt dec)
- (Thread/sleep 10))
- (catch Exception _)
- (finally
- (try (.close out)
- (catch Exception _)))))
- {:status 200
- :headers {"Content-Type" "text/event-stream"
- "Cache-Control" "no-cache"}
- :body in}))
- server (jetty/run-jetty handler {:port 0
- :join? false
- :output-buffer-size 1})
- url (str "http://localhost:" (.. server getURI getPort))]
- (try
- (let [res (http/request {:method :post :url url :as :stream})
- reducible (self.core/sse-reducible (:body res))
- ;; Take only 2 items, which should trigger early termination
- result (reduce (fn [acc item]
- (if (< (count acc) 2)
- (conj acc item)
- (reduced acc)))
- []
- reducible)]
- (is (= ["msg-0" "msg-1"] result))
- ;; Give time for cancellation to propagate
- (Thread/sleep 30)
- ;; Should have stopped writing when reduction stopped
- (is (> @cnt 20) "SHOULD have stopped writing when reduction terminated early"))
- (finally
- (.stop server)))))
-
(deftest lite-aisdk-xf-test
(testing "streams text deltas immediately instead of batching"
(let [chunks [{:type :start :messageId "msg-1"}
diff --git a/test/metabase/metabot/test_util.clj b/test/metabase/metabot/test_util.clj
index fa638f4fac6..50da4f006be 100644
--- a/test/metabase/metabot/test_util.clj
+++ b/test/metabase/metabot/test_util.clj
@@ -6,8 +6,13 @@
[metabase.metabot.tools :as metabot.tools]
[metabase.metabot.tools.core :as tools.core]
[metabase.metabot.tools.error :as tools.error]
+ [metabase.util :as u]
[metabase.util.json :as json]
- [metabase.util.log :as log]))
+ [metabase.util.log :as log])
+ (:import
+ (java.io ByteArrayOutputStream InputStream IOException OutputStream)
+ (java.net InetAddress ServerSocket Socket SocketException)
+ (java.util.zip GZIPOutputStream)))
(set! *warn-on-reflection* true)
@@ -229,3 +234,222 @@
:display-name "snip"
:snippet-id Integer/MAX_VALUE}}}
{:lib/type :mbql.stage/mbql}]})
+
+;;; ──────────────────────────────────────────────────────────────────
+;;; Chunked SSE fake server
+;;;
+;;; A raw-socket LLM server that streams SSE with `Transfer-Encoding: chunked`, for tests that must see when
+;;; the client drops the connection. Apache HttpClient drains the rest of a chunked body when the body
+;;; closes: https://github.com/dakrone/clj-http/issues/627
+;;; ──────────────────────────────────────────────────────────────────
+
+(defn- read-head!
+ "Read an HTTP request head from `in`, up to the blank line, and return its lines. Nil at EOF."
+ [^InputStream in]
+ (let [sb (StringBuilder.)]
+ (loop []
+ (let [b (.read in)]
+ (cond
+ (neg? b) nil
+ (str/ends-with? (.append sb (char b)) "\r\n\r\n") (str/split-lines (str sb))
+ :else (recur))))))
+
+(defn- header-value
+ "The value of the header `header-name` (lower case) in the head `lines`, or nil."
+ [lines header-name]
+ (some (fn [line]
+ (let [[k v] (str/split line #":\s*" 2)]
+ (when (= header-name (u/lower-case-en k))
+ v)))
+ lines))
+
+(defn- read-request!
+ "Read one request from `in`. Return its `:method`, its `:path` and `:max-tokens?`, true when the JSON body
+ has a `max_tokens` key. Nil at EOF."
+ [^InputStream in]
+ (when-let [[request-line & header-lines] (read-head! in)]
+ (let [[method path] (str/split request-line #" ")
+ length (parse-long (or (header-value header-lines "content-length") "0"))
+ body (String. (.readNBytes in (int length)) "UTF-8")]
+ {:method method
+ :path path
+ :max-tokens? (str/includes? body "\"max_tokens\"")})))
+
+(defn- write-ascii!
+ [^OutputStream out ^String s]
+ (.write out (.getBytes s "US-ASCII")))
+
+(defn- write-chunk!
+ "Write `data` to `out` as one HTTP chunk, and flush it. Write nothing for empty `data`."
+ [^OutputStream out ^bytes data]
+ (when (pos? (alength data))
+ (write-ascii! out (str (Integer/toHexString (alength data)) "\r\n"))
+ (.write out data)
+ (write-ascii! out "\r\n")
+ (.flush out)))
+
+(defn- plain-encoder
+ "An encoder that sends text as UTF-8 bytes."
+ []
+ {:encode (fn [^String s] (.getBytes s "UTF-8"))
+ :finish (fn [] (byte-array 0))})
+
+(defn- gzip-encoder
+ "An encoder that sends text as one gzip stream. It flushes after each piece, so the client can read it at once."
+ []
+ (let [buf (ByteArrayOutputStream.)
+ gz (GZIPOutputStream. buf true)
+ drain! (fn [] (let [data (.toByteArray buf)] (.reset buf) data))]
+ {:encode (fn [^String s] (.write gz (.getBytes s "UTF-8")) (.flush gz) (drain!))
+ :finish (fn [] (.finish gz) (drain!))}))
+
+(defn- response-head
+ [gzip?]
+ (str "HTTP/1.1 200 OK\r\n"
+ "Content-Type: text/event-stream\r\n"
+ "Transfer-Encoding: chunked\r\n"
+ "Connection: close\r\n"
+ (when gzip? "Content-Encoding: gzip\r\n")
+ "\r\n"))
+
+(defn- default-event
+ "The `i`th event of a stream: an OpenAI Chat Completions text delta."
+ [i]
+ {:id "chatcmpl-fake"
+ :model "fake-model"
+ :choices [{:index 0
+ :delta {:role "assistant" :content (str "chunk-" i " ")}
+ :finish_reason nil}]})
+
+(defn- write-events!
+ "Write `n` SSE events, one every `interval-ms`. Call `on-sent` with the count after each event."
+ [write! n {:keys [interval-ms event-fn]} on-sent]
+ (dotimes [i n]
+ (write! (str "data: " (json/encode (event-fn i)) "\n\n"))
+ (on-sent (inc i))
+ (Thread/sleep (long interval-ms))))
+
+(defn- write-after!
+ "Write what comes after the events: `[DONE]` for `:done`, nothing for `:eof`, and for `:keep-alive` `[DONE]`
+ and then 20 s of `: ping` comments."
+ [write! after]
+ (case after
+ :done (write! "data: [DONE]\n\n")
+ :eof nil
+ :keep-alive (do (write! "data: [DONE]\n\n")
+ (dotimes [_ 2000]
+ (write! ": ping\n\n")
+ (Thread/sleep 10)))))
+
+(defn- stream-response!
+ "Write a chunked SSE response of `n` events to `out`. Throws an `IOException` when the client is gone."
+ [^OutputStream out n {:keys [gzip? after] :as opts} on-sent]
+ (let [{:keys [encode finish]} (if gzip? (gzip-encoder) (plain-encoder))
+ write! #(write-chunk! out (encode %))]
+ (write-ascii! out (response-head gzip?))
+ (.flush out)
+ (write-events! write! n opts on-sent)
+ (write-after! write! after)
+ (write-chunk! out (finish))
+ (write-ascii! out "0\r\n\r\n")
+ (.flush out)))
+
+(defn- event-count
+ "How many events the fake sends for `request`: `:short-total` for a capped (`max_tokens`) call, else `:total`."
+ [{:keys [max-tokens?]} {:keys [total short-total]}]
+ (if (and short-total max-tokens?) short-total total))
+
+(defn- add-connection!
+ "Add the record of a new connection for `request` to `connections`, and return its index."
+ [connections request]
+ (dec (count (swap! connections conj (assoc request :sent 0 :disconnected-at-ms nil)))))
+
+(defn- serve-post!
+ "Stream the response to one `POST`. The first `POST` also reports to the server's `:sent` and `:disconnected`."
+ [^OutputStream out request opts {:keys [connections sent disconnected first-post]}]
+ (let [idx (add-connection! connections request)
+ first? (compare-and-set! first-post false true)
+ count! (fn [n]
+ (swap! connections assoc-in [idx :sent] n)
+ (when first? (reset! sent n)))]
+ (try
+ (stream-response! out (event-count request opts) opts count!)
+ (catch IOException _
+ (swap! connections assoc-in [idx :disconnected-at-ms] (System/currentTimeMillis))
+ (when first?
+ (deliver disconnected (get-in @connections [idx :sent])))))))
+
+(defn- serve-not-found!
+ [^OutputStream out request {:keys [connections]}]
+ (add-connection! connections request)
+ (write-ascii! out "HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
+ (.flush out))
+
+(defn- serve-connection!
+ "Serve the one request of `socket`: a `POST` gets the stream, any other method a 404."
+ [^Socket socket opts state]
+ (with-open [socket socket]
+ (let [out (.getOutputStream socket)
+ request (read-request! (.getInputStream socket))]
+ (cond
+ (nil? request) nil
+ (= "POST" (:method request)) (serve-post! out request opts state)
+ :else (serve-not-found! out request state)))))
+
+(defn- accept-loop!
+ "Accept connections on `server` until it closes, and serve each one on its own thread."
+ [^ServerSocket server opts {:keys [sockets] :as state}]
+ (future
+ (try
+ (loop []
+ (let [socket (.accept server)]
+ (swap! sockets conj socket)
+ (future
+ (try
+ (serve-connection! socket opts state)
+ (catch Exception e
+ (log/debug e "Fake SSE server: a connection failed"))))
+ (recur)))
+ (catch SocketException _
+ nil))))
+
+(defn start-chunked-sse-server!
+ "Start a fake LLM server on localhost that answers each `POST` with a chunked SSE stream.
+
+ Options (all optional):
+
+ :total - events per stream (default 10)
+ :interval-ms - pause after each event (default 10)
+ :event-fn - fn of the event index that returns the event map (default: an OpenAI Chat Completions
+ text delta)
+ :after - what follows the events: `:done` (`data: [DONE]`, the default), `:eof` (nothing) or
+ `:keep-alive` (`[DONE]`, then 20 s of `: ping` comments). The chunked terminator follows.
+ :gzip? - send the body with `Content-Encoding: gzip`
+ :port - the port to listen on (default 0, a free port)
+ :short-total - events for a `POST` whose JSON body has `max_tokens`, such as a title call
+
+ Any other method gets a 404. Returns:
+
+ :url - the base URL, e.g. `http://127.0.0.1:52011`
+ :sent - atom: the events sent on the first `POST`
+ :disconnected - promise: the `:sent` count when a write to the first `POST` failed (the client left)
+ :connections - atom: one map per request, `{:method :path :max-tokens? :sent :disconnected-at-ms}`
+ :stop! - fn that closes the server and every open connection"
+ ([]
+ (start-chunked-sse-server! {}))
+ ([opts]
+ (let [opts (merge {:total 10 :interval-ms 10 :event-fn default-event :after :done :port 0} opts)
+ server (ServerSocket. (int (:port opts)) 50 (InetAddress/getLoopbackAddress))
+ state {:connections (atom [])
+ :sent (atom 0)
+ :disconnected (promise)
+ :first-post (atom false)
+ :sockets (atom [])}]
+ (accept-loop! server opts state)
+ {:url (str "http://127.0.0.1:" (.getLocalPort server))
+ :sent (:sent state)
+ :disconnected (:disconnected state)
+ :connections (:connections state)
+ :stop! (fn []
+ (.close server)
+ (run! #(.close ^Socket %) @(:sockets state)))})))