Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Report error class and code on agent_used_tool (#85735)

lbrdnkb8364ea2945e
diff --git a/src/metabase/analytics/snowplow.clj b/src/metabase/analytics/snowplow.clj
index d6e7c724600..6b1eee8bc0f 100644
--- a/src/metabase/analytics/snowplow.clj
+++ b/src/metabase/analytics/snowplow.clj
@@ -217,8 +217,10 @@
[:result {:optional true} :string]
[:event-details {:optional true}
[:map {:closed true}
- ["tool_name" {:optional true} [:maybe :string]]
- ["step" {:optional true} :int]]]])
+ ["tool_name" {:optional true} [:maybe :string]]
+ ["step" {:optional true} :int]
+ ["error_class" {:optional true} :string]
+ ["error_code" {:optional true} :string]]]])
(def SnowplowEventData
"Closed shape of the `data` payload accepted by [[track-event!]], as a union of the shapes each `SnowplowSchema`
diff --git a/src/metabase/metabot/self.clj b/src/metabase/metabot/self.clj
index dab2b71f0a7..a8cfd7da0ef 100644
--- a/src/metabase/metabot/self.clj
+++ b/src/metabase/metabot/self.clj
@@ -283,30 +283,36 @@
(defn- report-tool-usage-xf
"Transducer that fires an agent_used_tool :snowplow/ai_service_event per tool call.
Only fires when :source and :request-id are present in tracking-opts. A tool name outside `tools` is
- model output that may carry user data, so it is reported as \"unknown\"."
+ model output that may carry user data, so it is reported as \"unknown\".
+
+ A failed call also reports its error's class and code from `tools.runtime/invoke`.
+ The codes are namespaced keywords, so moving or renaming one changes the reported value.
+ They describe the whole call, not the failed items inside a batched call.
+ The message and the recovery text never leave the instance."
[{:keys [request-id session-id source profile-id iteration]} tools]
(map (fn [part]
(when (and (some? source)
(some? request-id)
(= (:type part) :tool-output))
- (analytics.core/track-event! :snowplow/ai_service_event
- {:hashed-metabase-license-token (analytics.core/hashed-metabase-token-or-uuid)
- :request-id (analytics.core/uuid->ai-service-hex-uuid request-id)
- :source source
- :event "agent_used_tool"
- :user-id api/*current-user-id*
- :session-id session-id
- :profile (some-> profile-id name)
- :duration-ms (some-> (:duration-ms part) long)
- ;; A failed call carries `:error` on its result — success
- ;; is the absence of it, as it is everywhere else.
- :result (if (get-in part [:result :error])
- "error"
- "success")
- :event-details (cond-> {"tool_name" (if (contains? tools (:function part))
- (:function part)
- "unknown")}
- (some? iteration) (assoc "step" iteration))}))
+ (let [{:keys [class code] :as error} (get-in part [:result :error])]
+ (analytics.core/track-event! :snowplow/ai_service_event
+ {:hashed-metabase-license-token (analytics.core/hashed-metabase-token-or-uuid)
+ :request-id (analytics.core/uuid->ai-service-hex-uuid request-id)
+ :source source
+ :event "agent_used_tool"
+ :user-id api/*current-user-id*
+ :session-id session-id
+ :profile (some-> profile-id name)
+ :duration-ms (some-> (:duration-ms part) long)
+ ;; A failed call carries `:error` on its result — success
+ ;; is the absence of it, as it is everywhere else.
+ :result (if error "error" "success")
+ :event-details (cond-> {"tool_name" (if (contains? tools (:function part))
+ (:function part)
+ "unknown")}
+ (some? iteration) (assoc "step" iteration)
+ error (assoc "error_class" (name class)
+ "error_code" (u/qualified-name code)))})))
part)))
(defn- with-retries
diff --git a/test/metabase/metabot/agent/core_test.clj b/test/metabase/metabot/agent/core_test.clj
index 410f94e9749..dbe153a39f7 100644
--- a/test/metabase/metabot/agent/core_test.clj
+++ b/test/metabase/metabot/agent/core_test.clj
@@ -1520,8 +1520,12 @@
:tracking-opts {:session-id "00000000-0000-0000-0000-000000000002"}})
(let [events (snowplow-test/pop-event-data-and-user-id!)
tool-events (filter #(= "agent_used_tool" (get-in % [:data "event"])) events)]
- (is (=? [{:data {"event" "agent_used_tool"
- "result" "error"}}]
+ (is (=? [{:data {"event" "agent_used_tool"
+ "result" "error"
+ "event_details" {"tool_name" "search"
+ "step" 1
+ "error_class" "validation"
+ "error_code" "invalid-arguments"}}}]
tool-events)))))))))))
(deftest token-usage-snowplow-test
diff --git a/test/metabase/metabot/self_test.clj b/test/metabase/metabot/self_test.clj
index 95b8c28feb8..ad03cdbe7e9 100644
--- a/test/metabase/metabot/self_test.clj
+++ b/test/metabase/metabot/self_test.clj
@@ -2088,7 +2088,9 @@
"event_details" {"tool_name" "get-time"}}}
{:data {"event" "agent_used_tool"
"result" "error"
- "event_details" {"tool_name" "unknown"}}}]
+ "event_details" {"tool_name" "unknown"
+ "error_class" "validation"
+ "error_code" "unknown-tool"}}}]
tool-events))
(is (= 2 (count (filter #(= ":metabot.agent/run-tool" (:name %)) spans))))
(is (some #(str/includes? (:message %) "call-2") logs))
@@ -2910,17 +2912,24 @@
(testing (str "from `invoke` through the part to the SSE writer and persistence. Nothing spanned "
"this before, which is how the error moving into `:result` left three consumers "
"still reading it off the part and reporting every failure as a success.")
- (doseq [[label thrower expected-text]
+ (doseq [[label thrower expected-text expected-class expected-code]
[["recoverable"
#(tools.error/throw-recoverable! :metabase.metabot.tools.recoverable.common/not-found
{:kind :table :id 7})
- "Metabot could not complete this step."]
+ "Metabot could not complete this step."
+ "recoverable" "metabase.metabot.tools.recoverable.common/not-found"]
["unrecoverable with a :user-message"
#(tools.error/unrecoverable! ::nope {:user-message "Ask an admin for access."})
- "Ask an admin for access."]
+ "Ask an admin for access."
+ "unrecoverable" "metabase.metabot.self-test/nope"]
["unrecoverable without one"
#(tools.error/unrecoverable! ::nope {})
- "Metabot could not complete this step."]]]
+ "Metabot could not complete this step."
+ "unrecoverable" "metabase.metabot.self-test/nope"]
+ ["an exception nothing classified"
+ #(throw (ex-info "connection reset by peer" {}))
+ "Metabot could not complete this step."
+ "unrecoverable" "internal"]]]
(testing label
(let [outcome (failed-outcome "failer" thrower)
part (tool-output-part outcome)]
@@ -2936,6 +2945,27 @@
(testing "and never the model's own text, nor the EDN of the error map"
(is (not (str/includes? body "Don't retry")) body)
(is (not (str/includes? body ":class")) body))))
+ (testing "the Snowplow event carries the class and the code of the error, and no text of it"
+ (snowplow-test/with-fake-snowplow-collector
+ (into [] (#'self/report-tool-usage-xf (assoc snowplow-tracking-opts :profile-id :nlq :iteration 2)
+ #{"failer"})
+ [part])
+ ;; The collector may also hold a `new_instance_created` from the first read of a setting in
+ ;; this JVM, so filter for the tool events as the other Snowplow tests do.
+ (let [events (filter #(= "agent_used_tool" (get-in % [:data "event"]))
+ (snowplow-test/pop-event-data-and-user-id!))]
+ (is (=? [{:data {"event" "agent_used_tool"
+ "profile" "nlq"
+ "result" "error"
+ "event_details" {"tool_name" "failer"
+ "step" 2
+ "error_class" expected-class
+ "error_code" expected-code}}}]
+ events))
+ (is (not (str/includes? (pr-str events) (:output outcome)))
+ "the text written for the model stays on the instance")
+ (is (not (str/includes? (pr-str events) expected-text))
+ "so does the text written for the reader"))))
(testing (str "persistence stores it as a failed step, keeping the MODEL's text: "
"`tool-part->llm-messages` replays `:errorText` to the model, so a generic "
"sentence here would lose the authored failure on every replay")