The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Tell Metabot when it calls a tool it doesn't have (#82960)
Andrei Simionescu31a183a847de
diff --git a/src/metabase/metabot/self.clj b/src/metabase/metabot/self.cljindex c2b12ce4c0d..10ea0ccf861 100644--- a/src/metabase/metabot/self.clj+++ b/src/metabase/metabot/self.clj@@ -279,8 +279,9 @@(defn- report-tool-usage-xf"Transducer that fires an agent_used_tool :snowplow/ai_service_event per tool call.- Only fires when :source and :request-id are present in tracking-opts."- [{:keys [request-id session-id source profile-id iteration]}]+ Only fires when :source and :request-id are present in tracking-opts. A tool name outside `tools` is+ model output that may carry user data, so it is reported as \"unknown\"."+ [{:keys [request-id session-id source profile-id iteration]} tools](map (fn [part](when (and (some? source)(some? request-id)@@ -295,7 +296,9 @@:profile (some-> profile-id name):duration-ms (some-> (:duration-ms part) long):result (if (:error part) "error" "success")- :event-details (cond-> {"tool_name" (:function part)}+ :event-details (cond-> {"tool_name" (if (contains? tools (:function part))+ (:function part)+ "unknown")}(some? iteration) (assoc "step" iteration))}))part)))@@ -465,7 +468,7 @@(core/stamp-tool-titles-xf tools)(report-aisdk-errors-xf tracking-opts)(report-token-usage-xf tracking-opts)- (report-tool-usage-xf tracking-opts))+ (report-tool-usage-xf tracking-opts tools))(stream-fn streaming-opts)))](reify clojure.lang.IReduceInit(reduce [_ rf init]diff --git a/src/metabase/metabot/self/core.clj b/src/metabase/metabot/self/core.cljindex 3d6b54254dd..0fc7493256b 100644--- a/src/metabase/metabot/self/core.clj+++ b/src/metabase/metabot/self/core.clj@@ -937,45 +937,60 @@environment — `mu/defn` only instruments dev and test namespaces — and amismatch is returned to the model as a repair-oriented error.+ A call to a tool outside `tools` gets an error listing the ones it can call. Its name is model output,+ so logs and span data record it as \"unknown\".+Chunks have a ::duration-ms key added for internal use which is not part of the aisdk spec."- [tool-call-id tool-name tool chunks]- (ait/with-tool-call {:ai/tool-name tool-name- :ai/tool-call-id tool-call-id}- (with-span :info {:name :metabot.agent/run-tool- :tool-name tool-name- :tool-call-id tool-call-id}- (let [start-ms (u/start-timer)- assoc-ms (fn [duration-ms]- (fn [chunk]- (cond-> chunk- (= (:type chunk) :tool-output-available) (assoc ::duration-ms duration-ms))))- results (try- (let [{:keys [arguments]} (into {} (aisdk-xf) chunks)- arguments (walk/keywordize-keys (or (coerce-stringified-json arguments) {}))- arguments (coerce-stringified-scalars tool arguments)- decode (tool-decode-fn tool)- arguments (cond-> arguments decode decode)- _ (validate-tool-arguments! tool arguments)]- (log/debug "Executing tool" {:tool-name tool-name})- (when (ait/capture-active?)- (ait/record! {:ai/tool-args arguments}))- (let [tool-fn (tool-call-fn tool)- result (tool-fn arguments)]- (log/debug "Tool returned" {:tool-name tool-name :result-type (type result)})- (collect-tool-result tool-call-id tool-name result)))- (catch Exception e- (if (:agent-error? (ex-data e))- (log/debugf "Tool %s: agent validation error: %s" tool-name (ex-message e))- (log/warn "Tool execution failed" {:tool-name tool-name :error (ex-message e)}))- [{:type :tool-output-available- :toolCallId tool-call-id- :toolName tool-name- :error {:message (concise-tool-error e)- :type (str (type e))}}]))]- (when (ait/capture-active?)- (ait/record! {:ai/tool-output results}))- (mapv (assoc-ms (u/since-ms start-ms))- results)))))+ [tool-call-id tool-name tools chunks]+ (let [tool (get tools tool-name)+ safe-name (if tool tool-name "unknown")]+ (ait/with-tool-call {:ai/tool-name tool-name+ :ai/tool-call-id tool-call-id}+ (with-span :info {:name :metabot.agent/run-tool+ :tool-name safe-name+ :tool-call-id tool-call-id}+ (let [start-ms (u/start-timer)+ assoc-ms (fn [duration-ms]+ (fn [chunk]+ (cond-> chunk+ (= (:type chunk) :tool-output-available) (assoc ::duration-ms duration-ms))))+ results (try+ (when-not tool+ (throw (ex-info (str "Tool `" tool-name "` does not exist. Available tools: "+ (str/join ", " (sort (keys tools))) ".")+ {:agent-error? true})))+ (let [{:keys [arguments]} (into {} (aisdk-xf) chunks)+ arguments (walk/keywordize-keys (or (coerce-stringified-json arguments) {}))+ arguments (coerce-stringified-scalars tool arguments)+ decode (tool-decode-fn tool)+ arguments (cond-> arguments decode decode)+ _ (validate-tool-arguments! tool arguments)]+ (log/debug "Executing tool" {:tool-name safe-name})+ (when (ait/capture-active?)+ (ait/record! {:ai/tool-args arguments}))+ (let [tool-fn (tool-call-fn tool)+ result (tool-fn arguments)]+ (log/debug "Tool returned" {:tool-name safe-name :result-type (type result)})+ (collect-tool-result tool-call-id tool-name result)))+ (catch Exception e+ (cond+ (nil? tool)+ (log/debugf "Tool call %s: unknown tool" tool-call-id)++ (:agent-error? (ex-data e))+ (log/debugf "Tool %s: agent validation error: %s" safe-name (ex-message e))++ :else+ (log/warn "Tool execution failed" {:tool-name safe-name :error (ex-message e)}))+ [{:type :tool-output-available+ :toolCallId tool-call-id+ :toolName tool-name+ :error {:message (concise-tool-error e)+ :type (str (type e))}}]))]+ (when (ait/capture-active?)+ (ait/record! {:ai/tool-output results}))+ (mapv (assoc-ms (u/since-ms start-ms))+ results))))))(defn tool-executor-xf"Transducer that executes tool calls in parallel on virtual threads.@@ -1004,8 +1019,7 @@([result {:keys [type toolCallId toolName] :as chunk}](case type:tool-input-start- (when (contains? tools toolName)- (vswap! active assoc toolCallId {:chunks [chunk]}))+ (vswap! active assoc toolCallId {:chunks [chunk]}):tool-input-delta(when (contains? @active toolCallId)@@ -1013,8 +1027,7 @@:tool-input-available(when-let [{:keys [chunks]} (get @active toolCallId)]- (let [tool (get tools toolName)- task (submit-virtual (bound-fn* #(run-tool toolCallId toolName tool chunks)))]+ (let [task (submit-virtual (bound-fn* #(run-tool toolCallId toolName tools chunks)))](vswap! active assoc toolCallId {:task task})));; otherwise: do nothingdiff --git a/test/metabase/metabot/self_test.clj b/test/metabase/metabot/self_test.cljindex ad8323c3993..12832d18d3c 100644--- a/test/metabase/metabot/self_test.clj+++ b/test/metabase/metabot/self_test.clj@@ -27,6 +27,7 @@[metabase.metabot.usage :as usage][metabase.test :as mt][metabase.test.fixtures :as fixtures]+ [metabase.tracing.test-util :as tracing.tu][metabase.util.http :as u.http][metabase.util.json :as json][metabase.util.log.capture :as log.capture]@@ -604,13 +605,18 @@(last result))))))(deftest ^:parallel tool-executor-xf-test-7- (testing "tool-executor-xf ignores unknown tool names"+ (testing "tool-executor-xf answers a call to an unknown tool with an error listing the tools it can call"(let [chunks (test-util/parts->aisdk-chunks[{:type :start :id "msg-789"}- {:type :tool-input :id "call-1" :function "unknown-tool" :arguments {:foo :bar}}])+ {:type :tool-input :id "call-1" :function "analyze_chart" :arguments {:chart_config_id "abc"}}])result (into [] (self.core/tool-executor-xf test-util/TOOLS) chunks)]- (is (= chunks result)- "Unknown tools should be ignored, chunks pass through unchanged"))))+ (is (=? (conj (vec chunks)+ {:type :tool-output-available+ :toolCallId "call-1"+ :toolName "analyze_chart"+ :error {:message (str "Tool `analyze_chart` does not exist. "+ "Available tools: convert-currency, get-time, mock-llm, no-arg.")}})+ result)))));;; tool argument validation tests@@ -1609,7 +1615,7 @@(deftest call-llm-snowplow-test(llm.tu/with-default-connections- (testing "fires :snowplow/token_usage and :snowplow/ai_service_event for call-llm with a tool call"+ (testing "fires :snowplow/token_usage and :snowplow/ai_service_event for call-llm with tool calls"(let [rasta-id (mt/user->id :rasta)];; The adapter pre-sums input + cache_creation + cache_read into :promptTokens,;; so the mock supplies the already-summed value (950 = 100 fresh + 50 cache_creation + 800 cache_read).@@ -1619,6 +1625,8 @@[{:type :start :id "msg-1"}{:type :tool-input :id "call-1" :function "get-time":arguments {:tz "UTC"}}+ {:type :tool-input :id "call-2" :function "lookup_jane_doe_4165551234"+ :arguments {}}{:type :usage :usage {:promptTokens 950:completionTokens 20:cacheCreationTokens 50@@ -1626,8 +1634,11 @@:model "test-model" :id "msg-1"}]))](mt/with-current-user rasta-id(snowplow-test/with-fake-snowplow-collector- (run! identity (self/call-llm "openrouter/test-model" nil [] test-util/TOOLS snowplow-tracking-opts))- (let [events (snowplow-test/pop-event-data-and-user-id!)+ (let [[spans logs] (tracing.tu/with-span-exporter [exporter]+ (log.capture/with-log-messages-for-level [messages [metabase.metabot.self.core :debug]]+ (run! identity (self/call-llm "openrouter/test-model" nil [] test-util/TOOLS snowplow-tracking-opts))+ [(tracing.tu/finished-spans exporter) (messages)]))+ events (snowplow-test/pop-event-data-and-user-id!)token-events (filter #(contains? (:data %) "total_tokens") events)tool-events (filter #(= "agent_used_tool" (get-in % [:data "event"])) events)](is (=? [{:user-id (str rasta-id)@@ -1649,8 +1660,15 @@"result" "success""duration_ms" nat-int?"session_id" "00000000-0000-0000-0000-000000000002"- "event_details" {"tool_name" "get-time"}}}]- tool-events))))))))))+ "event_details" {"tool_name" "get-time"}}}+ {:data {"event" "agent_used_tool"+ "result" "error"+ "event_details" {"tool_name" "unknown"}}}]+ tool-events))+ (is (= 2 (count (filter #(= ":metabot.agent/run-tool" (:name %)) spans))))+ (is (some #(str/includes? (:message %) "call-2") logs))+ (is (not (str/includes? (pr-str [events spans logs]) "jane_doe"))+ "a name the model made up never reaches analytics, spans or logs")))))))))(deftest call-llm-structured-snowplow-test(llm.tu/with-default-connections