The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Flush held link text only before tool calls, results and errors (#83933)
Andrei Simionescu0ffca4925151
diff --git a/src/metabase/metabot/agent/markdown_link_buffer.clj b/src/metabase/metabot/agent/markdown_link_buffer.cljindex a80519d3901..9137a78920a 100644--- a/src/metabase/metabot/agent/markdown_link_buffer.clj+++ b/src/metabase/metabot/agent/markdown_link_buffer.clj@@ -118,7 +118,7 @@1. Accumulates queries/charts from tool-output parts (via :structured-output)2. Buffers markdown links to handle links split across text chunks3. Resolves metabase:// URLs to proper Metabase paths- 4. Flushes buffered content where the text ends: before the next part other than text or usage, and at stream end+ 4. Flushes buffered content where the text ends: before a tool call, a tool result or an error, and at stream endParameters:- initial-queries: Initial map of query-id to query data@@ -164,10 +164,9 @@(if (seq processed-text)(rf result (assoc part :text processed-text))result))- ;; Some servers send usage with every chunk, so a usage part doesn't end the text- :usage- (rf result part)+ (:tool-input :tool-output :error)(let [result (flush! result)](if (reduced? result)result- (rf result part)))))))))+ (rf result part)))+ (rf result part)))))))diff --git a/test/metabase/metabot/agent/core_test.clj b/test/metabase/metabot/agent/core_test.cljindex 6aab4b461b3..f0ffd1a2174 100644--- a/test/metabase/metabot/agent/core_test.clj+++ b/test/metabase/metabot/agent/core_test.clj@@ -727,6 +727,66 @@(is (=? {:type :text :text "The orders table has what you need."}(last (filter #(= :text (:type %)) result)))))))))))+(deftest next-call-ends-on-the-tool-result-test+ (testing "even when the text before the tool call ends like the start of a link"+ (mt/with-temporary-setting-values [llm-providers llm.tu/default-connections+ llm-metabot-provider test-provider]+ (let [call-count (atom 0)+ next-input (atom nil)]+ (mt/with-dynamic-fn-redefs [openrouter/openrouter (fn [{:keys [input]}]+ (if (= 1 (swap! call-count inc))+ (mut/mock-llm-response+ [{:type :text :id "t1" :text "Looking up [Orders"}+ {:type :tool-input+ :id "call-1"+ :function "search"+ :arguments {:keyword_queries ["orders"]}}])+ (do (reset! next-input input)+ (mut/mock-llm-response [{:type :text :text "Found it."}]))))+ metabot-search/search (constantly [])]+ (run-agent-loop! {:messages [{:role :user :content "Show me orders"}]+ :state {}+ :profile-id :internal+ :context {}})+ (is (=? {:type :tool-output :id "call-1"} (last @next-input))))))))++(deftest next-call-replays-link-split-by-reasoning-test+ (testing "reasoning between two text parts preserves the link and the tool result ends the next input"+ (mt/with-temporary-setting-values [llm-providers llm.tu/default-connections+ llm-metabot-provider test-provider]+ (let [call-count (atom 0)+ next-input (atom nil)]+ (mt/with-dynamic-fn-redefs [openrouter/openrouter (fn [{:keys [input]}]+ (if (= 1 (swap! call-count inc))+ [{:type :text-start :id "t1"}+ {:type :text-delta :id "t1" :delta "Looking up [Ord"}+ {:type :text-end :id "t1"}+ {:type :reasoning-start :id "r1"}+ {:type :reasoning-delta :id "r1" :delta "Thinking"}+ {:type :reasoning-end :id "r1"}+ {:type :text-start :id "t2"}+ {:type :text-delta :id "t2" :delta "ers](metabase://model/1)"}+ {:type :text-end :id "t2"}+ {:type :tool-input-start :toolCallId "call-1" :toolName "search"}+ {:type :tool-input-delta+ :toolCallId "call-1"+ :inputTextDelta (json/encode {:keyword_queries ["orders"]})}+ {:type :tool-input-available :toolCallId "call-1" :toolName "search"}]+ (do (reset! next-input input)+ (mut/mock-llm-response [{:type :text :text "Found it."}]))))+ metabot-search/search (constantly [])]+ (run-agent-loop! {:messages [{:role :user :content "Show me orders"}]+ :state {}+ :profile-id :internal+ :context {}})+ (is (=? [{:type :text :id "t1" :text "Looking up "}+ {:type :reasoning :id "r1" :text "Thinking"}+ {:type :text :id "t2" :text "[Orders](metabase://model/1)"}+ {:type :tool-input :id "call-1" :function "search" :arguments {:keyword_queries ["orders"]}}+ {:type :tool-output :id "call-1" :function "search"}]+ (filterv :type @next-input)))+ (is (=? {:type :tool-output :id "call-1"} (last @next-input))))))))+(deftest later-calls-resend-what-the-turn-already-sent-test(testing "edits made during a turn don't change the system prompt, tools or messages its earlier calls sent"(mt/with-temp [:model/Dashboard {dashboard-id :id} {:name "Ops original"}]diff --git a/test/metabase/metabot/agent/markdown_link_buffer_test.clj b/test/metabase/metabot/agent/markdown_link_buffer_test.cljindex 2cb309e6479..1eef5024536 100644--- a/test/metabase/metabot/agent/markdown_link_buffer_test.clj+++ b/test/metabase/metabot/agent/markdown_link_buffer_test.clj@@ -51,6 +51,11 @@(recur new-state more (conj outputs output)))[outputs (mlb/flush-state state) @registry])))))+(defn- resolve-parts+ "Run parts through a fresh [[mlb/resolve-xf]] with no queries or charts."+ [parts]+ (into [] (mlb/resolve-xf {} {} (atom {})) parts))+;;; State machine / buffering tests(deftest ^:parallel nested-bracket-handling-test@@ -289,22 +294,51 @@(is (= "metabase://query/q1" (first (vals @registry)))))))(deftest ^:parallel resolve-xf-flushes-held-text-where-the-text-ends-test- (let [resolve-parts #(into [] (mlb/resolve-xf {} {} (atom {})) %)]- (testing "text held for a possible link comes out before the tool call that follows it"- (is (=? [{:type :text :id "t1" :text "Let me check the "}- {:type :text :id "t1" :text "[Orders"}- {:type :tool-input :id "call-1"}- {:type :tool-output :id "call-1"}]- (resolve-parts [{:type :text :id "t1" :text "Let me check the [Orders"}- {:type :tool-input :id "call-1" :function "search" :arguments {}}- {:type :tool-output :id "call-1" :result {:output "Found ORDERS"}}]))))- (testing "a usage part between two chunks of a link doesn't split it"- (is (=? [{:type :text :text "See "}- {:type :usage}- {:type :text :text "[My Link](http://example.com)"}]- (resolve-parts [{:type :text :id "t1" :text "See [My "}- {:type :usage :usage {:promptTokens 1}}- {:type :text :id "t1" :text "Link](http://example.com)"}]))))))+ (doseq [[where after] [["before the tool call that follows it"+ [{:type :tool-input :id "call-1" :function "search" :arguments {}}+ {:type :tool-output :id "call-1" :result {:output "Found ORDERS"}}]]+ ["before an error"+ [{:type :error :error {:message "Stream failed"}}]]+ ["at the end of the stream"+ []]]]+ (testing (str "text held for a possible link comes out " where ", with the id of its text")+ (is (= (into [{:type :text :id "t1" :text "Let me check the "}+ {:type :text :id "t1" :text "[Orders"}]+ after)+ (resolve-parts (into [{:type :text :id "t1" :text "Let me check the [Orders"}] after)))))))++(deftest ^:parallel resolve-xf-supports-early-termination-test+ (let [take-parts (fn [n parts]+ (into [] (comp (mlb/resolve-xf {} {} (atom {})) (take n)) parts))]+ (testing "the consumer stops at the held text flushed before a tool call"+ (is (= [{:type :text :id "t1" :text "a "}+ {:type :text :id "t1" :text "[b"}]+ (take-parts 2 [{:type :text :id "t1" :text "a [b"}+ {:type :tool-input :id "c1"}+ {:type :tool-output :id "c1"}]))))+ (testing "the consumer stops before the held text flushed at the end of the stream"+ (is (= [{:type :text :id "t1" :text "a "}]+ (take-parts 1 [{:type :text :id "t1" :text "a [b"}]))))))++(deftest ^:parallel resolve-xf-holds-text-across-parts-that-do-not-end-it-test+ (doseq [part [{:type :usage :usage {:promptTokens 1}}+ {:type :reasoning :id "r1" :text "Thinking"}]]+ (testing (str "a " (name (:type part)) " part between two chunks of a link doesn't split it")+ (is (= [{:type :text :id "t1" :text "See "}+ part+ {:type :text :id "t1" :text "[My Model](/model/1)"}]+ (resolve-parts [{:type :text :id "t1" :text "See [My "}+ part+ {:type :text :id "t1" :text "Model](metabase://model/1)"}]))))))++(deftest ^:parallel resolve-xf-flushes-held-text-after-reasoning-before-tool-call-test+ (is (= [{:type :text :id "t1" :text "a "}+ {:type :reasoning :id "r1" :text "Thinking"}+ {:type :text :id "t1" :text "[b"}+ {:type :tool-input :id "c1"}]+ (resolve-parts [{:type :text :id "t1" :text "a [b"}+ {:type :reasoning :id "r1" :text "Thinking"}+ {:type :tool-input :id "c1"}]))))(deftest ^:parallel with-context-test(testing "updates state for subsequent link resolution"