Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Don't run Metabot tool calls from a model response that failed (#83259)

Andrei Simionescu813771b483ea
diff --git a/src/metabase/metabot/agent/core.clj b/src/metabase/metabot/agent/core.clj
index 88610c7c22b..b00d9cf5118 100644
--- a/src/metabase/metabot/agent/core.clj
+++ b/src/metabase/metabot/agent/core.clj
@@ -207,6 +207,12 @@
(= (:finish-reason %) "length"))
parts))
+(defn- errored?
+ "Whether this iteration's LLM call failed. A response that fails partway may contain a tool
+ call that never ran, so it must not continue the loop."
+ [parts]
+ (some #(= (:type %) :error) parts))
+
(defn- terminal-error-message
"Message from a tool failure no retry can fix (a permission denial), or nil if there was none."
[parts]
@@ -222,13 +228,15 @@
(and (< iteration max-iterations)
(has-tool-calls? parts)
(not (terminal-tool-call? terminal-tools parts))
- (not (truncated? parts))))
+ (not (truncated? parts))
+ (not (errored? parts))))
(defn- finish-reason
"Determine why the agent loop stopped."
[iteration max-iterations terminal-tools parts]
(cond
(truncated? parts) :length
+ (errored? parts) :error
(terminal-tool-call? terminal-tools parts) :terminal-tool
(and (>= iteration max-iterations)
(has-tool-calls? parts)) :max-iterations
diff --git a/src/metabase/metabot/self/core.clj b/src/metabase/metabot/self/core.clj
index 79cef25b584..a66e8b91147 100644
--- a/src/metabase/metabot/self/core.clj
+++ b/src/metabase/metabot/self/core.clj
@@ -1009,15 +1009,17 @@
"Transducer that executes tool calls in parallel on virtual threads.
Behavior:
- - Passes all chunks through unchanged as they arrive
+ - Passes chunks through unchanged as they arrive
- Tracks tool calls from :tool-input-start through :tool-input-available
- Spawns virtual thread for each tool when input is complete
+ - Once an :error chunk comes through, starts no more tools and drops the tool-input chunks after it
- At completion, waits for all tools and appends results
Tools can return: plain values, IReduceInit (reducible), or channels (legacy)."
[tools]
(fn [rf]
- (let [active (volatile! {})] ;; tool-call-id -> {:chunks [...]} or {:task derefable}
+ (let [active (volatile! {}) ;; tool-call-id -> {:chunks [...]} or {:task derefable}
+ errored? (volatile! false)]
(fn
([result]
(let [{tasks true
@@ -1030,22 +1032,32 @@
(rf result))))
([result {:keys [type toolCallId toolName] :as chunk}]
- (case type
- :tool-input-start
- (vswap! active assoc toolCallId {:chunks [chunk]})
-
- :tool-input-delta
- (when (contains? @active toolCallId)
- (vswap! active update-in [toolCallId :chunks] conj chunk))
-
- :tool-input-available
- (when-let [{:keys [chunks]} (get @active toolCallId)]
- (let [task (submit-virtual (bound-fn* #(run-tool toolCallId toolName tools chunks)))]
- (vswap! active assoc toolCallId {:task task})))
-
- ;; otherwise: do nothing
- nil)
- (rf result chunk))))))
+ (if (and @errored? (#{:tool-input-start :tool-input-delta :tool-input-available} type))
+ result
+ (do
+ (case type
+ :tool-input-start
+ (vswap! active assoc toolCallId {:chunks [chunk]})
+
+ :tool-input-delta
+ (when (contains? @active toolCallId)
+ (vswap! active update-in [toolCallId :chunks] conj chunk))
+
+ :tool-input-available
+ (when-let [{:keys [chunks]} (get @active toolCallId)]
+ (let [task (submit-virtual (bound-fn* #(run-tool toolCallId toolName tools chunks)))]
+ (vswap! active assoc toolCallId {:task task})))
+
+ :error
+ (let [cut-off (for [[id {:keys [task]}] @active :when (not task)] id)]
+ (vreset! errored? true)
+ (when (seq cut-off)
+ (log/warn "Dropping tool calls that a stream error cut off" {:tool-calls cut-off})
+ (vswap! active #(apply dissoc % cut-off))))
+
+ ;; otherwise: do nothing
+ nil)
+ (rf result chunk))))))))
(def ^:private max-body-preview-chars
"Cap on the body snippet spliced into provider error messages."
diff --git a/test/metabase/metabot/agent/core_test.clj b/test/metabase/metabot/agent/core_test.clj
index 0c3b050c13e..210e68ea19b 100644
--- a/test/metabase/metabot/agent/core_test.clj
+++ b/test/metabase/metabot/agent/core_test.clj
@@ -334,6 +334,23 @@
(is (some #(and (= :usage (:type %)) (= "length" (:finish-reason %))) result))
(is (some #(= :data (:type %)) result)
"state data part still closes the turn")))))
+ (testing "a turn whose stream failed stops the loop, even with a tool call present"
+ (let [call-count (atom 0)]
+ (mt/with-dynamic-fn-redefs [openrouter/openrouter (fn [_]
+ (swap! call-count inc)
+ (mut/mock-llm-response
+ [{:type :tool-input
+ :id "t1"
+ :function "search"
+ :arguments {:query "test"}}
+ {:type :error :errorText "Overloaded"}]))]
+ (let [result (into [] (agent/run-agent-loop
+ {:messages [{:role :user :content "Hi"}]
+ :state {}
+ :profile-id :embedding_next
+ :context {}}))]
+ (is (= 1 @call-count))
+ (is (= {:type :finish :finish-reason :error} (last result)))))))
(testing "handles errors gracefully"
(mt/with-dynamic-fn-redefs [openrouter/openrouter (fn [_]
(throw (ex-info "Mock error" {})))]
diff --git a/test/metabase/metabot/self/openai/chat_completions_test.clj b/test/metabase/metabot/self/openai/chat_completions_test.clj
index 8d3a4a3518a..1d5b752f26b 100644
--- a/test/metabase/metabot/self/openai/chat_completions_test.clj
+++ b/test/metabase/metabot/self/openai/chat_completions_test.clj
@@ -4,6 +4,8 @@
[metabase.metabot.self.core :as self.core]
[metabase.metabot.self.mistral :as mistral]
[metabase.metabot.self.openai.chat-completions :as chat-completions]
+ [metabase.metabot.self.openrouter :as openrouter]
+ [metabase.metabot.self.zai :as zai]
[metabase.metabot.test-util :as metabot.tu]
[metabase.util.json :as json]))
@@ -440,7 +442,7 @@
{:error {:message "Internal server error" :type "InternalServerError" :param nil :code 500}}
"Internal server error"]
["OpenRouter's error on a chunk that finishes the choice"
- (chat-completions/chat-completions->aisdk-chunks-xf)
+ (openrouter/openrouter->aisdk-chunks-xf)
{:id "cmpl-abc123"
:object "chat.completion.chunk"
:created 1234567890
@@ -456,6 +458,10 @@
:created 1702256327
:model "mistral-medium-3-5"
:choices [{:index 0 :delta {:content ""} :finish_reason "error"}]}
+ "The model provider failed to complete the response"]
+ ["Z.AI's network_error finish reason"
+ (zai/zai->aisdk-chunks-xf)
+ {:choices [{:index 0 :delta {} :finish_reason "network_error"}]}
"The model provider failed to complete the response"]]]
(testing shape
(testing "after closing the open text block"
diff --git a/test/metabase/metabot/self_test.clj b/test/metabase/metabot/self_test.clj
index d4aba51368f..0e7e736507d 100644
--- a/test/metabase/metabot/self_test.clj
+++ b/test/metabase/metabot/self_test.clj
@@ -642,6 +642,19 @@
"Available tools: convert-currency, get-time, mock-llm, no-arg.")}})
result)))))
+(deftest ^:parallel tool-executor-xf-stream-error-test
+ (testing "once an :error chunk comes through, a call still streaming never runs and a running one keeps its result"
+ (let [running (test-util/parts->aisdk-chunks
+ [{:type :tool-input :id "call-1" :function "get-time" :arguments {:tz "Europe/Kyiv"}}])
+ cut-off (test-util/parts->aisdk-chunks
+ [{:type :tool-input :id "call-2" :function "get-time" :arguments {:tz "Europe/Paris"}}])
+ streamed (concat running (butlast cut-off) [{:type :error :errorText "Overloaded"}])]
+ (log.capture/with-log-messages-for-level [messages [metabase.metabot.self.core :warn]]
+ (is (=? (conj (vec streamed) {:type :tool-output-available :toolCallId "call-1"})
+ (into [] (self.core/tool-executor-xf test-util/TOOLS) (concat streamed [(last cut-off)]))))
+ (testing "and a warning names the call that never ran"
+ (is (=? [{:level :warn :message #".*call-2.*"}] (messages))))))))
+
;;; tool argument validation tests
(defn- schema-tool