The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Keep made-up tool names out of stream error logs (#84272)
Andrei Simionescuc109845ba36c
diff --git a/src/metabase/metabot/self/core.clj b/src/metabase/metabot/self/core.cljindex 2445ff108b3..6beb76fa1d8 100644--- a/src/metabase/metabot/self/core.clj+++ b/src/metabase/metabot/self/core.clj@@ -390,10 +390,10 @@(nil? arguments) nil:else {::non-object-arguments arguments}))(catch Exception e- (log/warn "Failed to parse tool arguments as JSON, passing raw string"- {:tool (:toolName (first chunks))- :error (.getMessage e)- :raw-len (count raw)})+ (log/debug "Failed to parse tool arguments as JSON, passing raw string"+ {:tool-call-id (:toolCallId (first chunks))+ :error (.getMessage e)+ :raw-len (count raw)});; Return a sentinel map so the model gets a repair message from [[reject-unparsed-arguments!]];; rather than a cryptic JSON parse stacktrace.{::raw-arguments raw}))))@@ -1015,10 +1015,17 @@(fn [chunk](cond-> chunk(= (:type chunk) :tool-output-available) (assoc ::duration-ms duration-ms))))- outcome (if-not tool- (do (log/debugf "Tool call %s: unknown tool" tool-call-id)- (tools.runtime/unknown-tool-outcome tool-name (keys tools)))- (let [{:keys [arguments]} (into {} (aisdk-xf) chunks)]+ outcome (let [{:keys [arguments]} (into {} (aisdk-xf) chunks)+ raw (::raw-arguments arguments)]+ ;; Valid JSON can set `::raw-arguments` too, to a value that isn't a string.+ (when (string? raw)+ (log/warn "Failed to parse tool arguments as JSON"+ {:tool-name safe-name+ :tool-call-id tool-call-id+ :raw-len (count raw)}))+ (if-not tool+ (do (log/debugf "Tool call %s: unknown tool" tool-call-id)+ (tools.runtime/unknown-tool-outcome tool-name (keys tools)));; Checked before coercion: `coerce-stringified-json` would decode a;; double-encoded `{::non-object-arguments "{…}"}` into an object and;; garble the message.@@ -1089,12 +1096,14 @@(vswap! active assoc toolCallId {:task task}))):error- (let [cut-off (for [[id {:keys [task]}] @active :when (not task)] id)]+ (let [cut-off (u/for-map [[id {:keys [task chunks]}] @active+ :when (not task)+ :let [tool-name (:toolName (first chunks))]]+ [id (if (contains? tools tool-name) tool-name "unknown")])](vreset! errored? true)(when (seq cut-off)- (log/warn "Dropping tool calls that a stream error cut off"- {:tool-calls (into {} (map (juxt identity #(get-in @active [% :chunks 0 :toolName]))) cut-off)})- (vswap! active #(apply dissoc % cut-off))))+ (log/warn "Dropping tool calls that a stream error cut off" {:tool-calls cut-off})+ (vswap! active #(apply dissoc % (keys cut-off)))));; otherwise: do nothingnil)diff --git a/test/metabase/metabot/self_test.clj b/test/metabase/metabot/self_test.cljindex 84f9c235cd6..ba78b10d171 100644--- a/test/metabase/metabot/self_test.clj+++ b/test/metabase/metabot/self_test.clj@@ -724,13 +724,14 @@(let [running (test-util/parts->aisdk-chunks[{:type :tool-input :id "call-1" :function "get-time" :arguments {:tz "Europe/Kyiv"}}])cut-off (test-util/parts->aisdk-chunks- [{:type :tool-input :id "call-2" :function "get-time" :arguments {:tz "Europe/Paris"}}])+ [{:type :tool-input :id "call-2" :function "convert-currency"+ :arguments {:amount 100, :from "EUR", :to "USD"}}])streamed (concat running (butlast cut-off) [{:type :error :errorText "Overloaded"}])](log.capture/with-log-messages-for-level [messages [metabase.metabot.self.core :warn]](is (=? (conj (vec streamed) {:type :tool-output-available :toolCallId "call-1"})(into [] (self.core/tool-executor-xf test-util/TOOLS) (concat streamed [(last cut-off)]))))(testing "and a warning names the call that never ran"- (is (=? [{:level :warn :message #".*call-2.*get-time.*"}] (messages))))))))+ (is (=? [{:level :warn :message #".*call-2 convert-currency.*"}] (messages))))))));;; tool argument validation tests@@ -794,6 +795,32 @@(:output result)))(is (= {:class :validation :code :invalid-json} (:error result))))))+(deftest ^:parallel tool-unparseable-arguments-warning-test+ (testing "invalid JSON logs one warning with the tool name, the call ID and the length, but not the parse error"+ (let [raw "{\"tz\": jane_doe_4165551234}"]+ (doseq [[tool-name logged-name] [["get-time" "get-time"] ["analyze_chart" "unknown"]]]+ (testing tool-name+ (log.capture/with-log-messages-for-level [messages [metabase.metabot.self.core :warn]]+ (run! identity (eduction (self.core/tool-executor-xf test-util/TOOLS)+ (self.core/lite-aisdk-xf)+ [{:type :tool-input-start :toolName tool-name :toolCallId "call-j"}+ {:type :tool-input-delta :toolCallId "call-j" :inputTextDelta raw}+ {:type :tool-input-available :toolName tool-name :toolCallId "call-j"}]))+ (is (=? [{:level :warn+ :message (str "Failed to parse tool arguments as JSON {:tool-name " logged-name+ ", :tool-call-id call-j, :raw-len " (count raw) "}")}]+ (messages)))))))))++(deftest ^:parallel tool-sentinel-key-arguments-test+ (testing "valid JSON that sets the raw-arguments sentinel key gets a validation outcome"+ (doseq [[tool-name code] [["get-time" :invalid-json] ["analyze_chart" :unknown-tool]]]+ (testing tool-name+ (let [chunks (test-util/parts->aisdk-chunks+ [{:type :tool-input :id "call-k" :function tool-name+ :arguments {::self.core/raw-arguments 1}}])]+ (is (= {:class :validation :code code}+ (-> (into [] (self.core/tool-executor-xf test-util/TOOLS) chunks) last :result :error))))))))+(deftest ^:parallel tool-non-object-arguments-test(let [received (atom ::not-called)tools {"open" {:fn (fn [args] (reset! received args) {:output "ok"})@@ -2004,22 +2031,26 @@(deftest call-llm-snowplow-test(llm.tu/with-default-connections(testing "fires :snowplow/token_usage and :snowplow/ai_service_event for call-llm with tool calls"- (let [rasta-id (mt/user->id :rasta)]- ;; The adapter pre-sums input + cache_creation + cache_read into :promptTokens,- ;; so the mock supplies the already-summed value (950 = 100 fresh + 50 cache_creation + 800 cache_read).- ;; total_tokens reverts to prompt + completion = 950 + 20 = 970.+ (let [rasta-id (mt/user->id :rasta)+ ;; The adapter pre-sums input + cache_creation + cache_read into :promptTokens,+ ;; so the mock supplies the already-summed value (950 = 100 fresh + 50 cache_creation + 800 cache_read).+ ;; total_tokens reverts to prompt + completion = 950 + 20 = 970.+ chunks (test-util/parts->aisdk-chunks+ [{:type :start :id "msg-1"}+ {:type :tool-input :id "call-1" :function "get-time"+ :arguments {:tz "UTC"}}+ {:type :tool-input :id "call-2" :function "lookup_jane_doe_4165551234"+ :arguments {}}+ {:type :usage :usage {:promptTokens 950+ :completionTokens 20+ :cacheCreationTokens 50+ :cacheReadTokens 800}+ :model "test-model" :id "msg-1"}+ {:type :tool-input :id "call-3" :function "lookup_jane_doe_4165551234"+ :arguments {:tz "UTC"}}])](mt/with-dynamic-fn-redefs [openrouter/openrouter- (constantly (test-util/mock-llm-response- [{:type :start :id "msg-1"}- {:type :tool-input :id "call-1" :function "get-time"- :arguments {:tz "UTC"}}- {:type :tool-input :id "call-2" :function "lookup_jane_doe_4165551234"- :arguments {}}- {:type :usage :usage {:promptTokens 950- :completionTokens 20- :cacheCreationTokens 50- :cacheReadTokens 800}- :model "test-model" :id "msg-1"}]))]+ (constantly (concat (drop-last 2 chunks)+ [{:type :error :errorText "Overloaded"}]))](mt/with-current-user rasta-id(snowplow-test/with-fake-snowplow-collector(let [[spans logs] (tracing.tu/with-span-exporter [exporter]@@ -2057,6 +2088,8 @@tool-events))(is (= 2 (count (filter #(= ":metabot.agent/run-tool" (:name %)) spans))))(is (some #(str/includes? (:message %) "call-2") logs))+ (is (some #(str/includes? (:message %) "call-3 unknown") logs))+ (is (some #(str/includes? (:message %) ":tool-call-id call-3") logs))(is (not (str/includes? (pr-str [events spans logs]) "jane_doe"))"a name the model made up never reaches analytics, spans or logs")))))))))