The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Report error class and code on agent_used_tool (#85735)
lbrdnkb8364ea2945e
diff --git a/src/metabase/analytics/snowplow.clj b/src/metabase/analytics/snowplow.cljindex d6e7c724600..6b1eee8bc0f 100644--- a/src/metabase/analytics/snowplow.clj+++ b/src/metabase/analytics/snowplow.clj@@ -217,8 +217,10 @@[:result {:optional true} :string][:event-details {:optional true}[:map {:closed true}- ["tool_name" {:optional true} [:maybe :string]]- ["step" {:optional true} :int]]]])+ ["tool_name" {:optional true} [:maybe :string]]+ ["step" {:optional true} :int]+ ["error_class" {:optional true} :string]+ ["error_code" {:optional true} :string]]]])(def SnowplowEventData"Closed shape of the `data` payload accepted by [[track-event!]], as a union of the shapes each `SnowplowSchema`diff --git a/src/metabase/metabot/self.clj b/src/metabase/metabot/self.cljindex dab2b71f0a7..a8cfd7da0ef 100644--- a/src/metabase/metabot/self.clj+++ b/src/metabase/metabot/self.clj@@ -283,30 +283,36 @@(defn- report-tool-usage-xf"Transducer that fires an agent_used_tool :snowplow/ai_service_event per tool call.Only fires when :source and :request-id are present in tracking-opts. A tool name outside `tools` is- model output that may carry user data, so it is reported as \"unknown\"."+ model output that may carry user data, so it is reported as \"unknown\".++ A failed call also reports its error's class and code from `tools.runtime/invoke`.+ The codes are namespaced keywords, so moving or renaming one changes the reported value.+ They describe the whole call, not the failed items inside a batched call.+ The message and the recovery text never leave the instance."[{:keys [request-id session-id source profile-id iteration]} tools](map (fn [part](when (and (some? source)(some? request-id)(= (:type part) :tool-output))- (analytics.core/track-event! :snowplow/ai_service_event- {:hashed-metabase-license-token (analytics.core/hashed-metabase-token-or-uuid)- :request-id (analytics.core/uuid->ai-service-hex-uuid request-id)- :source source- :event "agent_used_tool"- :user-id api/*current-user-id*- :session-id session-id- :profile (some-> profile-id name)- :duration-ms (some-> (:duration-ms part) long)- ;; A failed call carries `:error` on its result — success- ;; is the absence of it, as it is everywhere else.- :result (if (get-in part [:result :error])- "error"- "success")- :event-details (cond-> {"tool_name" (if (contains? tools (:function part))- (:function part)- "unknown")}- (some? iteration) (assoc "step" iteration))}))+ (let [{:keys [class code] :as error} (get-in part [:result :error])]+ (analytics.core/track-event! :snowplow/ai_service_event+ {:hashed-metabase-license-token (analytics.core/hashed-metabase-token-or-uuid)+ :request-id (analytics.core/uuid->ai-service-hex-uuid request-id)+ :source source+ :event "agent_used_tool"+ :user-id api/*current-user-id*+ :session-id session-id+ :profile (some-> profile-id name)+ :duration-ms (some-> (:duration-ms part) long)+ ;; A failed call carries `:error` on its result — success+ ;; is the absence of it, as it is everywhere else.+ :result (if error "error" "success")+ :event-details (cond-> {"tool_name" (if (contains? tools (:function part))+ (:function part)+ "unknown")}+ (some? iteration) (assoc "step" iteration)+ error (assoc "error_class" (name class)+ "error_code" (u/qualified-name code)))})))part)))(defn- with-retriesdiff --git a/test/metabase/metabot/agent/core_test.clj b/test/metabase/metabot/agent/core_test.cljindex 410f94e9749..dbe153a39f7 100644--- a/test/metabase/metabot/agent/core_test.clj+++ b/test/metabase/metabot/agent/core_test.clj@@ -1520,8 +1520,12 @@:tracking-opts {:session-id "00000000-0000-0000-0000-000000000002"}})(let [events (snowplow-test/pop-event-data-and-user-id!)tool-events (filter #(= "agent_used_tool" (get-in % [:data "event"])) events)]- (is (=? [{:data {"event" "agent_used_tool"- "result" "error"}}]+ (is (=? [{:data {"event" "agent_used_tool"+ "result" "error"+ "event_details" {"tool_name" "search"+ "step" 1+ "error_class" "validation"+ "error_code" "invalid-arguments"}}}]tool-events)))))))))))(deftest token-usage-snowplow-testdiff --git a/test/metabase/metabot/self_test.clj b/test/metabase/metabot/self_test.cljindex 95b8c28feb8..ad03cdbe7e9 100644--- a/test/metabase/metabot/self_test.clj+++ b/test/metabase/metabot/self_test.clj@@ -2088,7 +2088,9 @@"event_details" {"tool_name" "get-time"}}}{:data {"event" "agent_used_tool""result" "error"- "event_details" {"tool_name" "unknown"}}}]+ "event_details" {"tool_name" "unknown"+ "error_class" "validation"+ "error_code" "unknown-tool"}}}]tool-events))(is (= 2 (count (filter #(= ":metabot.agent/run-tool" (:name %)) spans))))(is (some #(str/includes? (:message %) "call-2") logs))@@ -2910,17 +2912,24 @@(testing (str "from `invoke` through the part to the SSE writer and persistence. Nothing spanned ""this before, which is how the error moving into `:result` left three consumers ""still reading it off the part and reporting every failure as a success.")- (doseq [[label thrower expected-text]+ (doseq [[label thrower expected-text expected-class expected-code][["recoverable"#(tools.error/throw-recoverable! :metabase.metabot.tools.recoverable.common/not-found{:kind :table :id 7})- "Metabot could not complete this step."]+ "Metabot could not complete this step."+ "recoverable" "metabase.metabot.tools.recoverable.common/not-found"]["unrecoverable with a :user-message"#(tools.error/unrecoverable! ::nope {:user-message "Ask an admin for access."})- "Ask an admin for access."]+ "Ask an admin for access."+ "unrecoverable" "metabase.metabot.self-test/nope"]["unrecoverable without one"#(tools.error/unrecoverable! ::nope {})- "Metabot could not complete this step."]]]+ "Metabot could not complete this step."+ "unrecoverable" "metabase.metabot.self-test/nope"]+ ["an exception nothing classified"+ #(throw (ex-info "connection reset by peer" {}))+ "Metabot could not complete this step."+ "unrecoverable" "internal"]]](testing label(let [outcome (failed-outcome "failer" thrower)part (tool-output-part outcome)]@@ -2936,6 +2945,27 @@(testing "and never the model's own text, nor the EDN of the error map"(is (not (str/includes? body "Don't retry")) body)(is (not (str/includes? body ":class")) body))))+ (testing "the Snowplow event carries the class and the code of the error, and no text of it"+ (snowplow-test/with-fake-snowplow-collector+ (into [] (#'self/report-tool-usage-xf (assoc snowplow-tracking-opts :profile-id :nlq :iteration 2)+ #{"failer"})+ [part])+ ;; The collector may also hold a `new_instance_created` from the first read of a setting in+ ;; this JVM, so filter for the tool events as the other Snowplow tests do.+ (let [events (filter #(= "agent_used_tool" (get-in % [:data "event"]))+ (snowplow-test/pop-event-data-and-user-id!))]+ (is (=? [{:data {"event" "agent_used_tool"+ "profile" "nlq"+ "result" "error"+ "event_details" {"tool_name" "failer"+ "step" 2+ "error_class" expected-class+ "error_code" expected-code}}}]+ events))+ (is (not (str/includes? (pr-str events) (:output outcome)))+ "the text written for the model stays on the instance")+ (is (not (str/includes? (pr-str events) expected-text))+ "so does the text written for the reader"))))(testing (str "persistence stores it as a failed step, keeping the MODEL's text: ""`tool-part->llm-messages` replays `:errorText` to the model, so a generic ""sentence here would lose the authored failure on every replay")