Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Flush held link text only before tool calls, results and errors (#83933)

Andrei Simionescu0ffca4925151
diff --git a/src/metabase/metabot/agent/markdown_link_buffer.clj b/src/metabase/metabot/agent/markdown_link_buffer.clj
index a80519d3901..9137a78920a 100644
--- a/src/metabase/metabot/agent/markdown_link_buffer.clj
+++ b/src/metabase/metabot/agent/markdown_link_buffer.clj
@@ -118,7 +118,7 @@
1. Accumulates queries/charts from tool-output parts (via :structured-output)
2. Buffers markdown links to handle links split across text chunks
3. Resolves metabase:// URLs to proper Metabase paths
- 4. Flushes buffered content where the text ends: before the next part other than text or usage, and at stream end
+ 4. Flushes buffered content where the text ends: before a tool call, a tool result or an error, and at stream end
Parameters:
- initial-queries: Initial map of query-id to query data
@@ -164,10 +164,9 @@
(if (seq processed-text)
(rf result (assoc part :text processed-text))
result))
- ;; Some servers send usage with every chunk, so a usage part doesn't end the text
- :usage
- (rf result part)
+ (:tool-input :tool-output :error)
(let [result (flush! result)]
(if (reduced? result)
result
- (rf result part)))))))))
+ (rf result part)))
+ (rf result part)))))))
diff --git a/test/metabase/metabot/agent/core_test.clj b/test/metabase/metabot/agent/core_test.clj
index 6aab4b461b3..f0ffd1a2174 100644
--- a/test/metabase/metabot/agent/core_test.clj
+++ b/test/metabase/metabot/agent/core_test.clj
@@ -727,6 +727,66 @@
(is (=? {:type :text :text "The orders table has what you need."}
(last (filter #(= :text (:type %)) result)))))))))))
+(deftest next-call-ends-on-the-tool-result-test
+ (testing "even when the text before the tool call ends like the start of a link"
+ (mt/with-temporary-setting-values [llm-providers llm.tu/default-connections
+ llm-metabot-provider test-provider]
+ (let [call-count (atom 0)
+ next-input (atom nil)]
+ (mt/with-dynamic-fn-redefs [openrouter/openrouter (fn [{:keys [input]}]
+ (if (= 1 (swap! call-count inc))
+ (mut/mock-llm-response
+ [{:type :text :id "t1" :text "Looking up [Orders"}
+ {:type :tool-input
+ :id "call-1"
+ :function "search"
+ :arguments {:keyword_queries ["orders"]}}])
+ (do (reset! next-input input)
+ (mut/mock-llm-response [{:type :text :text "Found it."}]))))
+ metabot-search/search (constantly [])]
+ (run-agent-loop! {:messages [{:role :user :content "Show me orders"}]
+ :state {}
+ :profile-id :internal
+ :context {}})
+ (is (=? {:type :tool-output :id "call-1"} (last @next-input))))))))
+
+(deftest next-call-replays-link-split-by-reasoning-test
+ (testing "reasoning between two text parts preserves the link and the tool result ends the next input"
+ (mt/with-temporary-setting-values [llm-providers llm.tu/default-connections
+ llm-metabot-provider test-provider]
+ (let [call-count (atom 0)
+ next-input (atom nil)]
+ (mt/with-dynamic-fn-redefs [openrouter/openrouter (fn [{:keys [input]}]
+ (if (= 1 (swap! call-count inc))
+ [{:type :text-start :id "t1"}
+ {:type :text-delta :id "t1" :delta "Looking up [Ord"}
+ {:type :text-end :id "t1"}
+ {:type :reasoning-start :id "r1"}
+ {:type :reasoning-delta :id "r1" :delta "Thinking"}
+ {:type :reasoning-end :id "r1"}
+ {:type :text-start :id "t2"}
+ {:type :text-delta :id "t2" :delta "ers](metabase://model/1)"}
+ {:type :text-end :id "t2"}
+ {:type :tool-input-start :toolCallId "call-1" :toolName "search"}
+ {:type :tool-input-delta
+ :toolCallId "call-1"
+ :inputTextDelta (json/encode {:keyword_queries ["orders"]})}
+ {:type :tool-input-available :toolCallId "call-1" :toolName "search"}]
+ (do (reset! next-input input)
+ (mut/mock-llm-response [{:type :text :text "Found it."}]))))
+ metabot-search/search (constantly [])]
+ (run-agent-loop! {:messages [{:role :user :content "Show me orders"}]
+ :state {}
+ :profile-id :internal
+ :context {}})
+ (is (=? [{:type :text :id "t1" :text "Looking up "}
+ {:type :reasoning :id "r1" :text "Thinking"}
+ {:type :text :id "t2" :text "[Orders](metabase://model/1)"}
+ {:type :tool-input :id "call-1" :function "search" :arguments {:keyword_queries ["orders"]}}
+ {:type :tool-output :id "call-1" :function "search"}]
+ (filterv :type @next-input)))
+ (is (=? {:type :tool-output :id "call-1"} (last @next-input))))))))
+
(deftest later-calls-resend-what-the-turn-already-sent-test
(testing "edits made during a turn don't change the system prompt, tools or messages its earlier calls sent"
(mt/with-temp [:model/Dashboard {dashboard-id :id} {:name "Ops original"}]
diff --git a/test/metabase/metabot/agent/markdown_link_buffer_test.clj b/test/metabase/metabot/agent/markdown_link_buffer_test.clj
index 2cb309e6479..1eef5024536 100644
--- a/test/metabase/metabot/agent/markdown_link_buffer_test.clj
+++ b/test/metabase/metabot/agent/markdown_link_buffer_test.clj
@@ -51,6 +51,11 @@
(recur new-state more (conj outputs output)))
[outputs (mlb/flush-state state) @registry])))))
+(defn- resolve-parts
+ "Run parts through a fresh [[mlb/resolve-xf]] with no queries or charts."
+ [parts]
+ (into [] (mlb/resolve-xf {} {} (atom {})) parts))
+
;;; State machine / buffering tests
(deftest ^:parallel nested-bracket-handling-test
@@ -289,22 +294,51 @@
(is (= "metabase://query/q1" (first (vals @registry)))))))
(deftest ^:parallel resolve-xf-flushes-held-text-where-the-text-ends-test
- (let [resolve-parts #(into [] (mlb/resolve-xf {} {} (atom {})) %)]
- (testing "text held for a possible link comes out before the tool call that follows it"
- (is (=? [{:type :text :id "t1" :text "Let me check the "}
- {:type :text :id "t1" :text "[Orders"}
- {:type :tool-input :id "call-1"}
- {:type :tool-output :id "call-1"}]
- (resolve-parts [{:type :text :id "t1" :text "Let me check the [Orders"}
- {:type :tool-input :id "call-1" :function "search" :arguments {}}
- {:type :tool-output :id "call-1" :result {:output "Found ORDERS"}}]))))
- (testing "a usage part between two chunks of a link doesn't split it"
- (is (=? [{:type :text :text "See "}
- {:type :usage}
- {:type :text :text "[My Link](http://example.com)"}]
- (resolve-parts [{:type :text :id "t1" :text "See [My "}
- {:type :usage :usage {:promptTokens 1}}
- {:type :text :id "t1" :text "Link](http://example.com)"}]))))))
+ (doseq [[where after] [["before the tool call that follows it"
+ [{:type :tool-input :id "call-1" :function "search" :arguments {}}
+ {:type :tool-output :id "call-1" :result {:output "Found ORDERS"}}]]
+ ["before an error"
+ [{:type :error :error {:message "Stream failed"}}]]
+ ["at the end of the stream"
+ []]]]
+ (testing (str "text held for a possible link comes out " where ", with the id of its text")
+ (is (= (into [{:type :text :id "t1" :text "Let me check the "}
+ {:type :text :id "t1" :text "[Orders"}]
+ after)
+ (resolve-parts (into [{:type :text :id "t1" :text "Let me check the [Orders"}] after)))))))
+
+(deftest ^:parallel resolve-xf-supports-early-termination-test
+ (let [take-parts (fn [n parts]
+ (into [] (comp (mlb/resolve-xf {} {} (atom {})) (take n)) parts))]
+ (testing "the consumer stops at the held text flushed before a tool call"
+ (is (= [{:type :text :id "t1" :text "a "}
+ {:type :text :id "t1" :text "[b"}]
+ (take-parts 2 [{:type :text :id "t1" :text "a [b"}
+ {:type :tool-input :id "c1"}
+ {:type :tool-output :id "c1"}]))))
+ (testing "the consumer stops before the held text flushed at the end of the stream"
+ (is (= [{:type :text :id "t1" :text "a "}]
+ (take-parts 1 [{:type :text :id "t1" :text "a [b"}]))))))
+
+(deftest ^:parallel resolve-xf-holds-text-across-parts-that-do-not-end-it-test
+ (doseq [part [{:type :usage :usage {:promptTokens 1}}
+ {:type :reasoning :id "r1" :text "Thinking"}]]
+ (testing (str "a " (name (:type part)) " part between two chunks of a link doesn't split it")
+ (is (= [{:type :text :id "t1" :text "See "}
+ part
+ {:type :text :id "t1" :text "[My Model](/model/1)"}]
+ (resolve-parts [{:type :text :id "t1" :text "See [My "}
+ part
+ {:type :text :id "t1" :text "Model](metabase://model/1)"}]))))))
+
+(deftest ^:parallel resolve-xf-flushes-held-text-after-reasoning-before-tool-call-test
+ (is (= [{:type :text :id "t1" :text "a "}
+ {:type :reasoning :id "r1" :text "Thinking"}
+ {:type :text :id "t1" :text "[b"}
+ {:type :tool-input :id "c1"}]
+ (resolve-parts [{:type :text :id "t1" :text "a [b"}
+ {:type :reasoning :id "r1" :text "Thinking"}
+ {:type :tool-input :id "c1"}]))))
(deftest ^:parallel with-context-test
(testing "updates state for subsequent link resolution"