The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Don't run Metabot tool calls from a model response that failed (#83259)
Andrei Simionescu813771b483ea
diff --git a/src/metabase/metabot/agent/core.clj b/src/metabase/metabot/agent/core.cljindex 88610c7c22b..b00d9cf5118 100644--- a/src/metabase/metabot/agent/core.clj+++ b/src/metabase/metabot/agent/core.clj@@ -207,6 +207,12 @@(= (:finish-reason %) "length"))parts))+(defn- errored?+ "Whether this iteration's LLM call failed. A response that fails partway may contain a tool+ call that never ran, so it must not continue the loop."+ [parts]+ (some #(= (:type %) :error) parts))+(defn- terminal-error-message"Message from a tool failure no retry can fix (a permission denial), or nil if there was none."[parts]@@ -222,13 +228,15 @@(and (< iteration max-iterations)(has-tool-calls? parts)(not (terminal-tool-call? terminal-tools parts))- (not (truncated? parts))))+ (not (truncated? parts))+ (not (errored? parts))))(defn- finish-reason"Determine why the agent loop stopped."[iteration max-iterations terminal-tools parts](cond(truncated? parts) :length+ (errored? parts) :error(terminal-tool-call? terminal-tools parts) :terminal-tool(and (>= iteration max-iterations)(has-tool-calls? parts)) :max-iterationsdiff --git a/src/metabase/metabot/self/core.clj b/src/metabase/metabot/self/core.cljindex 79cef25b584..a66e8b91147 100644--- a/src/metabase/metabot/self/core.clj+++ b/src/metabase/metabot/self/core.clj@@ -1009,15 +1009,17 @@"Transducer that executes tool calls in parallel on virtual threads.Behavior:- - Passes all chunks through unchanged as they arrive+ - Passes chunks through unchanged as they arrive- Tracks tool calls from :tool-input-start through :tool-input-available- Spawns virtual thread for each tool when input is complete+ - Once an :error chunk comes through, starts no more tools and drops the tool-input chunks after it- At completion, waits for all tools and appends resultsTools can return: plain values, IReduceInit (reducible), or channels (legacy)."[tools](fn [rf]- (let [active (volatile! {})] ;; tool-call-id -> {:chunks [...]} or {:task derefable}+ (let [active (volatile! {}) ;; tool-call-id -> {:chunks [...]} or {:task derefable}+ errored? (volatile! false)](fn([result](let [{tasks true@@ -1030,22 +1032,32 @@(rf result))))([result {:keys [type toolCallId toolName] :as chunk}]- (case type- :tool-input-start- (vswap! active assoc toolCallId {:chunks [chunk]})-- :tool-input-delta- (when (contains? @active toolCallId)- (vswap! active update-in [toolCallId :chunks] conj chunk))-- :tool-input-available- (when-let [{:keys [chunks]} (get @active toolCallId)]- (let [task (submit-virtual (bound-fn* #(run-tool toolCallId toolName tools chunks)))]- (vswap! active assoc toolCallId {:task task})))-- ;; otherwise: do nothing- nil)- (rf result chunk))))))+ (if (and @errored? (#{:tool-input-start :tool-input-delta :tool-input-available} type))+ result+ (do+ (case type+ :tool-input-start+ (vswap! active assoc toolCallId {:chunks [chunk]})++ :tool-input-delta+ (when (contains? @active toolCallId)+ (vswap! active update-in [toolCallId :chunks] conj chunk))++ :tool-input-available+ (when-let [{:keys [chunks]} (get @active toolCallId)]+ (let [task (submit-virtual (bound-fn* #(run-tool toolCallId toolName tools chunks)))]+ (vswap! active assoc toolCallId {:task task})))++ :error+ (let [cut-off (for [[id {:keys [task]}] @active :when (not task)] id)]+ (vreset! errored? true)+ (when (seq cut-off)+ (log/warn "Dropping tool calls that a stream error cut off" {:tool-calls cut-off})+ (vswap! active #(apply dissoc % cut-off))))++ ;; otherwise: do nothing+ nil)+ (rf result chunk))))))))(def ^:private max-body-preview-chars"Cap on the body snippet spliced into provider error messages."diff --git a/test/metabase/metabot/agent/core_test.clj b/test/metabase/metabot/agent/core_test.cljindex 0c3b050c13e..210e68ea19b 100644--- a/test/metabase/metabot/agent/core_test.clj+++ b/test/metabase/metabot/agent/core_test.clj@@ -334,6 +334,23 @@(is (some #(and (= :usage (:type %)) (= "length" (:finish-reason %))) result))(is (some #(= :data (:type %)) result)"state data part still closes the turn")))))+ (testing "a turn whose stream failed stops the loop, even with a tool call present"+ (let [call-count (atom 0)]+ (mt/with-dynamic-fn-redefs [openrouter/openrouter (fn [_]+ (swap! call-count inc)+ (mut/mock-llm-response+ [{:type :tool-input+ :id "t1"+ :function "search"+ :arguments {:query "test"}}+ {:type :error :errorText "Overloaded"}]))]+ (let [result (into [] (agent/run-agent-loop+ {:messages [{:role :user :content "Hi"}]+ :state {}+ :profile-id :embedding_next+ :context {}}))]+ (is (= 1 @call-count))+ (is (= {:type :finish :finish-reason :error} (last result)))))))(testing "handles errors gracefully"(mt/with-dynamic-fn-redefs [openrouter/openrouter (fn [_](throw (ex-info "Mock error" {})))]diff --git a/test/metabase/metabot/self/openai/chat_completions_test.clj b/test/metabase/metabot/self/openai/chat_completions_test.cljindex 8d3a4a3518a..1d5b752f26b 100644--- a/test/metabase/metabot/self/openai/chat_completions_test.clj+++ b/test/metabase/metabot/self/openai/chat_completions_test.clj@@ -4,6 +4,8 @@[metabase.metabot.self.core :as self.core][metabase.metabot.self.mistral :as mistral][metabase.metabot.self.openai.chat-completions :as chat-completions]+ [metabase.metabot.self.openrouter :as openrouter]+ [metabase.metabot.self.zai :as zai][metabase.metabot.test-util :as metabot.tu][metabase.util.json :as json]))@@ -440,7 +442,7 @@{:error {:message "Internal server error" :type "InternalServerError" :param nil :code 500}}"Internal server error"]["OpenRouter's error on a chunk that finishes the choice"- (chat-completions/chat-completions->aisdk-chunks-xf)+ (openrouter/openrouter->aisdk-chunks-xf){:id "cmpl-abc123":object "chat.completion.chunk":created 1234567890@@ -456,6 +458,10 @@:created 1702256327:model "mistral-medium-3-5":choices [{:index 0 :delta {:content ""} :finish_reason "error"}]}+ "The model provider failed to complete the response"]+ ["Z.AI's network_error finish reason"+ (zai/zai->aisdk-chunks-xf)+ {:choices [{:index 0 :delta {} :finish_reason "network_error"}]}"The model provider failed to complete the response"]]](testing shape(testing "after closing the open text block"diff --git a/test/metabase/metabot/self_test.clj b/test/metabase/metabot/self_test.cljindex d4aba51368f..0e7e736507d 100644--- a/test/metabase/metabot/self_test.clj+++ b/test/metabase/metabot/self_test.clj@@ -642,6 +642,19 @@"Available tools: convert-currency, get-time, mock-llm, no-arg.")}})result)))))+(deftest ^:parallel tool-executor-xf-stream-error-test+ (testing "once an :error chunk comes through, a call still streaming never runs and a running one keeps its result"+ (let [running (test-util/parts->aisdk-chunks+ [{:type :tool-input :id "call-1" :function "get-time" :arguments {:tz "Europe/Kyiv"}}])+ cut-off (test-util/parts->aisdk-chunks+ [{:type :tool-input :id "call-2" :function "get-time" :arguments {:tz "Europe/Paris"}}])+ streamed (concat running (butlast cut-off) [{:type :error :errorText "Overloaded"}])]+ (log.capture/with-log-messages-for-level [messages [metabase.metabot.self.core :warn]]+ (is (=? (conj (vec streamed) {:type :tool-output-available :toolCallId "call-1"})+ (into [] (self.core/tool-executor-xf test-util/TOOLS) (concat streamed [(last cut-off)]))))+ (testing "and a warning names the call that never ran"+ (is (=? [{:level :warn :message #".*call-2.*"}] (messages))))))))+;;; tool argument validation tests(defn- schema-tool