Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Report unexpected Metabot tool failures as tool errors (#82969)

Andrei Simionescu5375ffed93c1
diff --git a/src/metabase/metabot/self/core.clj b/src/metabase/metabot/self/core.clj
index 0fc7493256b..07104eb35f0 100644
--- a/src/metabase/metabot/self/core.clj
+++ b/src/metabase/metabot/self/core.clj
@@ -981,7 +981,7 @@
(log/debugf "Tool %s: agent validation error: %s" safe-name (ex-message e))
:else
- (log/warn "Tool execution failed" {:tool-name safe-name :error (ex-message e)}))
+ (log/error e "Tool execution failed" {:tool-name safe-name}))
[{:type :tool-output-available
:toolCallId tool-call-id
:toolName tool-name
diff --git a/src/metabase/metabot/tools/autogen_dashboard.clj b/src/metabase/metabot/tools/autogen_dashboard.clj
index d24e04c5a18..7d4461d1caa 100644
--- a/src/metabase/metabot/tools/autogen_dashboard.clj
+++ b/src/metabase/metabot/tools/autogen_dashboard.clj
@@ -6,6 +6,7 @@
[metabase.metabot.db :as metabot.db]
[metabase.metabot.scope :as scope]
[metabase.metabot.tools.shared :as shared]
+ [metabase.metabot.tools.util :as metabot.tools.u]
[metabase.util.log :as log]
[metabase.util.malli :as mu]))
@@ -153,7 +154,4 @@
:instructions (:instructions result)}
(seq data-parts) (assoc :data-parts data-parts)))
(catch Exception e
- (log/errorf "Error creating autogenerated dashboard: %s" (ex-message e))
- (if (:agent-error? (ex-data e))
- {:output (ex-message e)}
- {:output (str "Failed to create autogenerated dashboard: " (or (ex-message e) "Unknown error"))}))))
+ (metabot.tools.u/handle-agent-error e))))
diff --git a/src/metabase/metabot/tools/charts.clj b/src/metabase/metabot/tools/charts.clj
index 432db464425..f09d9ccd7cd 100644
--- a/src/metabase/metabot/tools/charts.clj
+++ b/src/metabase/metabot/tools/charts.clj
@@ -10,7 +10,7 @@
[metabase.metabot.tools.shared :as shared]
[metabase.metabot.tools.shared.instructions :as instructions]
[metabase.metabot.tools.shared.llm-shape :as llm-shape]
- [metabase.util.log :as log]
+ [metabase.metabot.tools.util :as metabot.tools.u]
[metabase.util.malli :as mu]))
(set! *warn-on-reflection* true)
@@ -60,10 +60,7 @@
:title title
:description description})]})
(catch Exception e
- (log/errorf "Error creating chart: %s" (ex-message e))
- (if (:agent-error? (ex-data e))
- {:output (ex-message e)}
- {:output (str "Failed to create chart: " (or (ex-message e) "Unknown error"))}))))
+ (metabot.tools.u/handle-agent-error e))))
(def ^:private edit-chart-schema
[:map {:closed true}
@@ -109,7 +106,4 @@
:title title
:description description})]})
(catch Exception e
- (log/errorf "Error editing chart: %s" (ex-message e))
- (if (:agent-error? (ex-data e))
- {:output (ex-message e)}
- {:output (str "Failed to edit chart: " (or (ex-message e) "Unknown error"))}))))
+ (metabot.tools.u/handle-agent-error e))))
diff --git a/src/metabase/metabot/tools/clarification.clj b/src/metabase/metabot/tools/clarification.clj
index 278ee491f8f..f10a05d9c41 100644
--- a/src/metabase/metabot/tools/clarification.clj
+++ b/src/metabase/metabot/tools/clarification.clj
@@ -45,9 +45,6 @@
[{:keys [question options]} :- [:map {:closed true}
[:question :string]
[:options {:optional true} [:maybe [:sequential :string]]]]]
- (try
- (let [result (ask-for-sql-clarification {:question question
- :options options})]
- (assoc result :output (format-clarification-output (:structured-output result))))
- (catch Exception e
- {:output (str "Failed to ask clarification: " (or (ex-message e) "Unknown error"))})))
+ (let [result (ask-for-sql-clarification {:question question
+ :options options})]
+ (assoc result :output (format-clarification-output (:structured-output result)))))
diff --git a/src/metabase/metabot/tools/create_alert.clj b/src/metabase/metabot/tools/create_alert.clj
index 2eaf152dc8a..4b16302b219 100644
--- a/src/metabase/metabot/tools/create_alert.clj
+++ b/src/metabase/metabot/tools/create_alert.clj
@@ -11,7 +11,6 @@
[metabase.metabot.tools.util :as metabot.tools.u]
[metabase.notification.api :as notification.api]
[metabase.util.cron :as u.cron]
- [metabase.util.log :as log]
[metabase.util.malli :as mu]))
(set! *warn-on-reflection* true)
@@ -134,10 +133,10 @@
`send_once` is optional (default false); when true the alert is deleted after it fires once."
[{:keys [card_id send_condition schedule send_once]} :- alert-schema]
(let [slack-channel-id (:slack_channel_id (shared/current-context))]
- (when-not slack-channel-id
- (throw (ex-info "This tool can only be used from a Slack channel"
- {:agent-error? true})))
(try
+ (when-not slack-channel-id
+ (throw (ex-info "This tool can only be used from a Slack channel"
+ {:agent-error? true})))
(let [result (create-alert {:card-id card_id
:send-condition (keyword send_condition)
:schedule schedule
@@ -147,5 +146,4 @@
{:output (:error result)}
{:output (or (:output result) "Alert created successfully.")}))
(catch Exception e
- (log/errorf "Failed to create alert: %s" (ex-message e))
- {:output (str "Failed to create alert: " (or (ex-message e) "Unknown error"))}))))
+ (metabot.tools.u/handle-agent-or-api-error e)))))
diff --git a/src/metabase/metabot/tools/create_dashboard_subscription.clj b/src/metabase/metabot/tools/create_dashboard_subscription.clj
index 4211e26576e..5d0b52f96be 100644
--- a/src/metabase/metabot/tools/create_dashboard_subscription.clj
+++ b/src/metabase/metabot/tools/create_dashboard_subscription.clj
@@ -9,7 +9,6 @@
[metabase.metabot.tools.shared :as shared]
[metabase.metabot.tools.util :as metabot.tools.u]
[metabase.pulse.api :as pulse.api]
- [metabase.util.log :as log]
[metabase.util.malli :as mu]
[toucan2.core :as t2]))
@@ -95,10 +94,10 @@
[:dashboard_id :int]
[:schedule tools.create-alert/schedule-schema]]]
(let [slack-channel-id (:slack_channel_id (shared/current-context))]
- (when-not slack-channel-id
- (throw (ex-info "This tool can only be used from a Slack channel"
- {:agent-error? true})))
(try
+ (when-not slack-channel-id
+ (throw (ex-info "This tool can only be used from a Slack channel"
+ {:agent-error? true})))
(let [result (create-dashboard-subscription
{:dashboard-id dashboard_id
:schedule schedule
@@ -107,5 +106,4 @@
{:output (:error result)}
{:output (or (:output result) "Dashboard subscription created successfully.")}))
(catch Exception e
- (log/errorf "Failed to create dashboard subscription: %s" (ex-message e))
- {:output (str "Failed to create dashboard subscription: " (or (ex-message e) "Unknown error"))}))))
+ (metabot.tools.u/handle-agent-or-api-error e)))))
diff --git a/src/metabase/metabot/tools/document.clj b/src/metabase/metabot/tools/document.clj
index d313a86c70a..f9144339caa 100644
--- a/src/metabase/metabot/tools/document.clj
+++ b/src/metabase/metabot/tools/document.clj
@@ -9,7 +9,6 @@
[metabase.metabot.tools.sql.create :as create-sql-query-tools]
[metabase.metabot.tools.util :as metabot.tools.u]
[metabase.query-processor.core :as qp]
- [metabase.util.log :as log]
[metabase.util.malli :as mu]
[metabase.warehouses.core :as warehouses]))
@@ -124,8 +123,7 @@
:structured-output {:database_id database-id
:sql_engine (:engine db)}})))
(catch Exception e
- (log/errorf "Error collecting document schema: %s" (ex-message e))
- {:output (str "Failed to collect schema: " (or (ex-message e) "Unknown error"))})))
+ (metabot.tools.u/handle-agent-or-api-error e))))
(def ^:private sql-chart-schema
[:map {:closed true}
@@ -175,10 +173,7 @@
{:output "Draft chart payload generated from SQL query."
:structured-output structured}))))
(catch Exception e
- (log/errorf "Error constructing SQL chart draft: %s" (ex-message e))
- (if (:agent-error? (ex-data e))
- (metabot.tools.u/handle-agent-error e)
- {:output (str "Failed to construct SQL chart draft: " (or (ex-message e) "Unknown error"))}))))
+ (metabot.tools.u/handle-agent-error e))))
(def ^:private model-chart-schema
"Schema for `document_construct_model_chart`. Mirrors `construct_notebook_query`'s
@@ -222,7 +217,4 @@
(or result
{:output "Failed to construct model chart draft."})))
(catch Exception e
- (log/errorf "Error constructing model chart draft: %s" (ex-message e))
- (if (:agent-error? (ex-data e))
- (metabot.tools.u/handle-agent-error e)
- {:output (str "Failed to construct model chart draft: " (or (ex-message e) "Unknown error"))}))))
+ (metabot.tools.u/handle-agent-error e))))
diff --git a/src/metabase/metabot/tools/metadata.clj b/src/metabase/metabot/tools/metadata.clj
index b587a391a03..d1ae1ee294f 100644
--- a/src/metabase/metabot/tools/metadata.clj
+++ b/src/metabase/metabot/tools/metadata.clj
@@ -11,6 +11,7 @@
[metabase.metabot.tools.shared :as shared]
[metabase.metabot.tools.shared.instructions :as instructions]
[metabase.metabot.tools.shared.llm-shape :as llm-shape]
+ [metabase.metabot.tools.util :as metabot.tools.u]
[metabase.util.i18n :refer [tru]]
[metabase.util.log :as log]
[metabase.util.malli :as mu]))
@@ -106,10 +107,7 @@
:metrics metrics
:errors errors}}))
(catch Exception e
- (log/errorf "Failed to fetch metadata: %s" (ex-message e))
- (if (:agent-error? (ex-data e))
- {:output (ex-message e)}
- {:output (str "Failed to fetch metadata: " (or (ex-message e) "Unknown error"))}))))
+ (metabot.tools.u/handle-agent-error e))))
(defn- format-with-instructions
[data instruction-text]
diff --git a/src/metabase/metabot/tools/resources.clj b/src/metabase/metabot/tools/resources.clj
index 149eaf1eb12..ab5cdd4dc12 100644
--- a/src/metabase/metabot/tools/resources.clj
+++ b/src/metabase/metabot/tools/resources.clj
@@ -81,6 +81,7 @@
[metabase.metabot.tools.shared.content-store :as shared.content-store]
[metabase.metabot.tools.shared.instructions :as instructions]
[metabase.metabot.tools.shared.llm-shape :as llm-shape]
+ [metabase.metabot.tools.util :as metabot.tools.u]
[metabase.models.interface :as mi]
[metabase.permissions.core :as perms]
[metabase.transforms.core :as transforms]
@@ -1021,7 +1022,7 @@
(str "Too many URIs provided (" (count uris) "). "
"Please limit to " max-concurrent-uris " URIs maximum. "
"Be more selective and focus on the most relevant items for the current task or fetch them in batches.")
- {:uri-count (count uris) :max max-concurrent-uris})))
+ {:agent-error? true :uri-count (count uris) :max max-concurrent-uris})))
;; Fetch all URIs (sequentially for now, could parallelize with pmap)
(let [resources (mapv fetch-single-uri uris)
@@ -1093,5 +1094,4 @@
(try
(read-resource {:uris uris})
(catch Exception e
- (log/errorf "Error in read_resource tool: %s" (ex-message e))
- {:output (str "Failed to read resources: " (or (ex-message e) "Unknown error"))})))
+ (metabot.tools.u/handle-agent-error e))))
diff --git a/src/metabase/metabot/tools/save_entity.clj b/src/metabase/metabot/tools/save_entity.clj
index 7a62feab40c..64adbc49955 100644
--- a/src/metabase/metabot/tools/save_entity.clj
+++ b/src/metabase/metabot/tools/save_entity.clj
@@ -18,10 +18,10 @@
[metabase.metabot.scope :as scope]
[metabase.metabot.tmpl :as te]
[metabase.metabot.tools.shared :as shared]
+ [metabase.metabot.tools.util :as metabot.tools.u]
[metabase.queries.core :as queries]
[metabase.query-permissions.core :as query-perms]
[metabase.util.i18n :refer [tru]]
- [metabase.util.log :as log]
[metabase.util.malli :as mu]
[toucan2.core :as t2]))
@@ -240,7 +240,4 @@
;; as the settled "Saved …" step label
:title (te/link question-name link)})]})
(catch Exception e
- (log/errorf "Error saving entity: %s" (ex-message e))
- (if (:agent-error? (ex-data e))
- {:output (ex-message e)}
- {:output (str "Failed to save: " (or (ex-message e) "Unknown error"))}))))
+ (metabot.tools.u/handle-agent-or-api-error e))))
diff --git a/src/metabase/metabot/tools/skills.clj b/src/metabase/metabot/tools/skills.clj
index 763b7fdb5d0..ea342404787 100644
--- a/src/metabase/metabot/tools/skills.clj
+++ b/src/metabase/metabot/tools/skills.clj
@@ -37,8 +37,4 @@
[{:keys [ids]} :- [:map {:closed true}
[:ids [:sequential
[:string {:description "A skill id, e.g. construct-notebook-query-core"}]]]]]
- (try
- {:output (str/join "\n\n" (map load-one ids))}
- (catch Exception e
- (log/errorf "Failed to load skill(s): %s" (ex-message e))
- {:output (str "Failed to load skill(s): " (or (ex-message e) "Unknown error"))})))
+ {:output (str/join "\n\n" (map load-one ids))})
diff --git a/src/metabase/metabot/tools/sql.clj b/src/metabase/metabot/tools/sql.clj
index a1593fc1e98..87093b991e0 100644
--- a/src/metabase/metabot/tools/sql.clj
+++ b/src/metabase/metabot/tools/sql.clj
@@ -13,7 +13,6 @@
[metabase.metabot.tools.sql.replace :as replace-sql-query-tools]
[metabase.metabot.tools.util :as metabot.tools.u]
[metabase.util :as u]
- [metabase.util.log :as log]
[metabase.util.malli :as mu]))
(set! *warn-on-reflection* true)
@@ -187,10 +186,7 @@
{:output (format-validation-error-output instr)
:instructions instr})))
(catch Exception e
- (log/errorf "Error editing SQL query: %s" (ex-message e))
- (if (:agent-error? (ex-data e))
- (metabot.tools.u/handle-agent-error e)
- {:output (str "Failed to edit SQL query: " (or (ex-message e) "Unknown error"))}))))
+ (metabot.tools.u/handle-agent-error e))))
;;; ──────────────────────────────────────────────────────────────────
;;; Replace SQL query
@@ -236,7 +232,4 @@
{:output (format-validation-error-output instr)
:instructions instr})))
(catch Exception e
- (log/errorf "Error replacing SQL query: %s" (ex-message e))
- (if (:agent-error? (ex-data e))
- (metabot.tools.u/handle-agent-error e)
- {:output (str "Failed to replace SQL query: " (or (ex-message e) "Unknown error"))}))))
+ (metabot.tools.u/handle-agent-error e))))
diff --git a/src/metabase/metabot/tools/subscriptions.clj b/src/metabase/metabot/tools/subscriptions.clj
index 16a9493f659..aefb8081142 100644
--- a/src/metabase/metabot/tools/subscriptions.clj
+++ b/src/metabase/metabot/tools/subscriptions.clj
@@ -6,9 +6,9 @@
[metabase.channel.settings :as channel.settings]
[metabase.metabot.db :as metabot.db]
[metabase.metabot.scope :as scope]
+ [metabase.metabot.tools.create-alert :as tools.create-alert]
[metabase.metabot.tools.util :as metabot.tools.u]
[metabase.pulse.api :as pulse.api]
- [metabase.util.log :as log]
[metabase.util.malli :as mu]
[toucan2.core :as t2]))
@@ -85,11 +85,7 @@
[:dashboard_id :int]
[:email {:optional true} [:maybe :string]]
[:slack_channel {:optional true} [:maybe :string]]
- [:schedule [:map {:closed true}
- [:frequency [:enum "hourly" "daily" "weekly" "monthly"]]
- [:hour {:optional true} [:maybe :int]]
- [:day_of_week {:optional true} [:maybe :string]]
- [:day_of_month {:optional true} [:maybe :string]]]]])
+ [:schedule tools.create-alert/schedule-schema]])
(mu/defn ^{:tool-name "create_dashboard_subscription"
:scope scope/agent-dashboard-subscribe}
@@ -105,19 +101,17 @@
email address."
[{:keys [dashboard_id email slack_channel schedule]} :- subscription-schema]
(try
- (create-dashboard-subscription
- {:dashboard-id dashboard_id
- :email email
- :slack-channel slack_channel
- :schedule (-> schedule
- (update :frequency keyword)
- (cond->
- (:day_of_week schedule) (-> (assoc :day-of-week (keyword (:day_of_week schedule)))
- (dissoc :day_of_week))
- (:day_of_month schedule) (-> (assoc :day-of-month (keyword (:day_of_month schedule)))
- (dissoc :day_of_month))))})
+ (-> (create-dashboard-subscription
+ {:dashboard-id dashboard_id
+ :email email
+ :slack-channel slack_channel
+ :schedule (-> schedule
+ (update :frequency keyword)
+ (cond->
+ (:day_of_week schedule) (-> (assoc :day-of-week (keyword (:day_of_week schedule)))
+ (dissoc :day_of_week))
+ (:day_of_month schedule) (-> (assoc :day-of-month (keyword (:day_of_month schedule)))
+ (dissoc :day_of_month))))})
+ (set/rename-keys {:error :output}))
(catch Exception e
- (log/errorf "Error creating dashboard subscription: %s" (ex-message e))
- (if (:agent-error? (ex-data e))
- {:output (ex-message e)}
- {:output (str "Failed to create dashboard subscription: " (or (ex-message e) "Unknown error"))}))))
+ (metabot.tools.u/handle-agent-or-api-error e))))
diff --git a/src/metabase/metabot/tools/todo.clj b/src/metabase/metabot/tools/todo.clj
index 41165bd0452..14728244d73 100644
--- a/src/metabase/metabot/tools/todo.clj
+++ b/src/metabase/metabot/tools/todo.clj
@@ -7,6 +7,7 @@
[metabase.metabot.agent.streaming :as streaming]
[metabase.metabot.scope :as scope]
[metabase.metabot.tools.shared :as shared]
+ [metabase.metabot.tools.util :as metabot.tools.u]
[metabase.util.log :as log]
[metabase.util.malli :as mu]))
@@ -128,9 +129,7 @@
:memory-atom shared/*memory-atom*})
format-todo-write-output)
(catch Exception e
- (if (:agent-error? (ex-data e))
- {:output (ex-message e)}
- {:output (str "Failed to update todo list: " (or (ex-message e) "Unknown error"))}))))
+ (metabot.tools.u/handle-agent-error e))))
(mu/defn ^{:tool-name "todo_read"
:scope scope/agent-todo-read}
@@ -138,9 +137,6 @@
"Read the current todo list from memory.
Returns the list of todos that have been created during this conversation."
[_args :- [:map {:closed true}]]
- (try
- (add-output
- (todo-read {:memory-atom shared/*memory-atom*})
- format-todo-read-output)
- (catch Exception e
- {:output (str "Failed to read todo list: " (or (ex-message e) "Unknown error"))})))
+ (add-output
+ (todo-read {:memory-atom shared/*memory-atom*})
+ format-todo-read-output))
diff --git a/src/metabase/metabot/tools/util.clj b/src/metabase/metabot/tools/util.clj
index 409d00a1069..8d2f96f0e26 100644
--- a/src/metabase/metabot/tools/util.clj
+++ b/src/metabase/metabot/tools/util.clj
@@ -20,6 +20,15 @@
terminal-error? (assoc :terminal-error? true))
(throw e))))
+(defn handle-agent-or-api-error
+ "Return an agent output for agent errors and API check refusals, re-throw `e` otherwise. A refusal is an exception
+ with a 4xx `:status-code`, like the 403 or 404 from [[api/read-check]]."
+ [e]
+ (let [{:keys [agent-error? status-code]} (ex-data e)]
+ (if (and (not agent-error?) (int? status-code) (<= 400 status-code 499))
+ {:output (ex-message e) :status-code status-code}
+ (handle-agent-error e))))
+
(defn convert-field-type
"Return tool type for `column`."
[column]
diff --git a/test/metabase/metabot/self_test.clj b/test/metabase/metabot/self_test.clj
index 12832d18d3c..7303d84e8dc 100644
--- a/test/metabase/metabot/self_test.clj
+++ b/test/metabase/metabot/self_test.clj
@@ -592,6 +592,21 @@
:type string?}}
(last result))))))
+(deftest ^:parallel tool-failure-log-level-test
+ (let [run-failing (fn [e]
+ (into [] (self.core/tool-executor-xf {"failing" {:fn (fn [_] (throw e))}})
+ (test-util/parts->aisdk-chunks
+ [{:type :tool-input :id "call-f" :function "failing" :arguments {}}])))]
+ (testing "an unexpected tool failure is logged at error with its exception"
+ (let [e (ex-info "boom" {})]
+ (log.capture/with-log-messages-for-level [messages [metabase.metabot.self.core :error]]
+ (run-failing e)
+ (is (= [[:error e]] (map (juxt :level :e) (messages)))))))
+ (testing "an agent error is not"
+ (log.capture/with-log-messages-for-level [messages [metabase.metabot.self.core :error]]
+ (run-failing (ex-info "No such field" {:agent-error? true}))
+ (is (empty? (messages)))))))
+
(deftest ^:parallel tool-executor-xf-test-6
(testing "tool-executor-xf handles nil arguments for no-arg tools"
(let [chunks (test-util/parts->aisdk-chunks
diff --git a/test/metabase/metabot/tools/autogen_dashboard_test.clj b/test/metabase/metabot/tools/autogen_dashboard_test.clj
index 3b7c87c5f23..b82803a7233 100644
--- a/test/metabase/metabot/tools/autogen_dashboard_test.clj
+++ b/test/metabase/metabot/tools/autogen_dashboard_test.clj
@@ -132,3 +132,12 @@
(autogen-dashboard/create-autogenerated-dashboard
{:source {:report_id 999999}
:memory-atom (atom {:state {}})})))))
+
+(deftest create-autogenerated-dashboard-tool-errors-test
+ (testing "a query missing from memory goes back to the agent as output"
+ (is (= {:output "Query not found in memory with ID: q-9. Available queries: []"}
+ (autogen-dashboard/create-autogenerated-dashboard-tool {:source {:query_id "q-9"}}))))
+ (testing "an unexpected error propagates to the agent loop"
+ (mt/with-dynamic-fn-redefs [autogen-dashboard/create-autogenerated-dashboard (fn [_] (throw (ex-info "boom" {})))]
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"boom"
+ (autogen-dashboard/create-autogenerated-dashboard-tool {:source {:table_id 1}}))))))
diff --git a/test/metabase/metabot/tools/charts_test.clj b/test/metabase/metabot/tools/charts_test.clj
index f5050dc27aa..b3721882b38 100644
--- a/test/metabase/metabot/tools/charts_test.clj
+++ b/test/metabase/metabot/tools/charts_test.clj
@@ -5,7 +5,10 @@
(:require
[clojure.test :refer :all]
[metabase.metabot.tools.charts :as charts]
- [metabase.metabot.tools.shared :as shared]))
+ [metabase.metabot.tools.charts.create :as create-chart-tools]
+ [metabase.metabot.tools.charts.edit :as edit-chart-tools]
+ [metabase.metabot.tools.shared :as shared]
+ [metabase.test :as mt]))
;; create-chart only needs the query present in queries-state; the link builder
;; json-encodes it and `->legacy-mbql` passes a non-MBQL 5 value through unchanged,
@@ -75,3 +78,21 @@
;; only looked at the chart's own (here empty) :queries, landing on nil.
(is (= {:query-id "q-1" :query stub-query}
(select-keys (:structured-output result) [:query-id :query])))))))
+
+(deftest chart-tools-errors-test
+ (testing "a missing query goes back to the agent as output"
+ (is (re-find #"^Query not found with ID: q-9"
+ (:output (binding [shared/*memory-atom* (atom {:state {:queries {}}})]
+ (charts/create-chart-tool {:data_source {:query_id "q-9"}
+ :viz_settings {:chart_type "bar"}
+ :title "Orders by month"
+ :description "Monthly count of orders."}))))))
+ (testing "an unexpected error propagates to the agent loop"
+ (mt/with-dynamic-fn-redefs [create-chart-tools/create-chart (fn [_] (throw (ex-info "boom" {})))
+ edit-chart-tools/edit-chart (fn [_] (throw (ex-info "boom" {})))]
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"boom" (run-create-chart)))
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"boom"
+ (charts/edit-chart-tool {:chart_id "c-1"
+ :new_viz_settings {:chart_type "pie"}
+ :title "Orders by month"
+ :description "Monthly count of orders."}))))))
diff --git a/test/metabase/metabot/tools/clarification_test.clj b/test/metabase/metabot/tools/clarification_test.clj
index 13438c75def..98b188212de 100644
--- a/test/metabase/metabot/tools/clarification_test.clj
+++ b/test/metabase/metabot/tools/clarification_test.clj
@@ -2,7 +2,8 @@
(:require
[clojure.string :as str]
[clojure.test :refer :all]
- [metabase.metabot.tools.clarification :as ask-clarification]))
+ [metabase.metabot.tools.clarification :as ask-clarification]
+ [metabase.test :as mt]))
(deftest ask-for-sql-clarification-test
(testing "returns structured-output (the success signal that lets a profile end the turn)"
@@ -36,3 +37,9 @@
"ask_for_sql_clarification must return :structured-output so the agent loop can stop")
(is (not (contains? result :final-response?))
"the tool-level :final-response? flag has been removed in favour of :terminal-tools"))))
+
+(deftest ask-for-sql-clarification-tool-unexpected-error-test
+ (testing "an unexpected error propagates to the agent loop"
+ (mt/with-dynamic-fn-redefs [ask-clarification/ask-for-sql-clarification (fn [_] (throw (ex-info "boom" {})))]
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"boom"
+ (ask-clarification/ask-for-sql-clarification-tool {:question "Which table?"}))))))
diff --git a/test/metabase/metabot/tools/create_alert_test.clj b/test/metabase/metabot/tools/create_alert_test.clj
index 93c5862b418..d21791f1ab3 100644
--- a/test/metabase/metabot/tools/create_alert_test.clj
+++ b/test/metabase/metabot/tools/create_alert_test.clj
@@ -4,6 +4,7 @@
[metabase.channel.settings :as channel.settings]
[metabase.metabot.tools.create-alert
:as metabot.tools.create-alert]
+ [metabase.metabot.tools.shared :as shared]
[metabase.test :as mt]))
(defn- alert-data [card-id]
@@ -74,3 +75,24 @@
channel.settings/slack-cached-channels-and-usernames (constantly {:channels []})]
(is (= {:error "no slack channel found with this name"}
(invoke-tool (assoc base-data :slack-channel "no-such-channel")))))))))
+
+(deftest create-alert-tool-errors-test
+ (let [create-alert! #(binding [shared/*memory-atom* (atom {:context {:slack_channel_id "C123"}})]
+ (metabot.tools.create-alert/create-alert-tool
+ {:card_id %
+ :send_condition "has_result"
+ :schedule {:frequency "daily" :hour 9}}))]
+ (testing "a call from outside Slack goes back to the agent as output"
+ (is (= {:output "This tool can only be used from a Slack channel"}
+ (metabot.tools.create-alert/create-alert-tool
+ {:card_id 1 :send_condition "has_result" :schedule {:frequency "daily" :hour 9}}))))
+ (testing "a card the user can't read goes back to the agent as output"
+ (mt/with-dynamic-fn-redefs [channel.settings/slack-configured? (constantly true)]
+ (mt/with-non-admin-groups-no-root-collection-perms
+ (mt/with-temp [:model/Card {card-id :id} {}]
+ (mt/with-current-user (mt/user->id :rasta)
+ (is (= "You don't have permissions to do that."
+ (:output (create-alert! card-id)))))))))
+ (testing "an unexpected error propagates to the agent loop"
+ (mt/with-dynamic-fn-redefs [metabot.tools.create-alert/create-alert (fn [_] (throw (ex-info "boom" {})))]
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"boom" (create-alert! 1)))))))
diff --git a/test/metabase/metabot/tools/create_dashboard_subscription_test.clj b/test/metabase/metabot/tools/create_dashboard_subscription_test.clj
new file mode 100644
index 00000000000..00f69c375a2
--- /dev/null
+++ b/test/metabase/metabot/tools/create_dashboard_subscription_test.clj
@@ -0,0 +1,10 @@
+(ns metabase.metabot.tools.create-dashboard-subscription-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase.metabot.tools.create-dashboard-subscription :as tools.create-dashboard-subscription]))
+
+(deftest ^:parallel slackbot-create-dashboard-subscription-outside-slack-test
+ (testing "a call from outside Slack goes back to the agent as output"
+ (is (= {:output "This tool can only be used from a Slack channel"}
+ (tools.create-dashboard-subscription/slackbot-create-dashboard-subscription-tool
+ {:dashboard_id 1 :schedule {:frequency "daily" :hour 9}})))))
diff --git a/test/metabase/metabot/tools/document_test.clj b/test/metabase/metabot/tools/document_test.clj
index a735b4f58aa..f1e9f42476a 100644
--- a/test/metabase/metabot/tools/document_test.clj
+++ b/test/metabase/metabot/tools/document_test.clj
@@ -160,3 +160,31 @@
structured (:structured-output result)]
(is (= chart-type (:display structured)))
(is (= chart-type (:chart_type structured))))))))
+
+(deftest document-tools-errors-test
+ (testing "a database that no longer exists goes back to the agent as output"
+ (mt/with-dynamic-fn-redefs [shared/current-context (fn [] {:references {"database:999999" "Gone"}})]
+ (mt/with-current-user (mt/user->id :crowberto)
+ (is (= "Not found." (:output (document-tools/document-schema-collect-tool {})))))))
+ (testing "an unexpected error propagates to the agent loop"
+ (mt/with-dynamic-fn-redefs [shared/current-context (fn [] {:references {"database:1" "Test Database"}})
+ warehouses/get-database (fn [_] (throw (ex-info "boom" {})))
+ create-sql-query-tools/create-sql-query (fn [_] (throw (ex-info "boom" {})))
+ construct-tools/construct-notebook-query-tool (fn [_] (throw (ex-info "boom" {})))]
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"boom"
+ (document-tools/document-schema-collect-tool {})))
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"boom"
+ (document-tools/document-construct-sql-chart-tool
+ {:database_id 1
+ :name "Test Name"
+ :description "Test Desc"
+ :analysis "Test Analysis"
+ :approach "Test Approach"
+ :sql "SELECT 1"
+ :viz_settings {:chart_type "bar"}})))
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"boom"
+ (document-tools/document-construct-model-chart-tool
+ {:name "Test Name"
+ :description "Test Desc"
+ :query ""
+ :viz_settings {:chart_type "bar"}}))))))
diff --git a/test/metabase/metabot/tools/metadata_test.clj b/test/metabase/metabot/tools/metadata_test.clj
index 6bafd8cd24a..79a1bb521ea 100644
--- a/test/metabase/metabot/tools/metadata_test.clj
+++ b/test/metabase/metabot/tools/metadata_test.clj
@@ -102,3 +102,12 @@
(is (= [] (:models structured-output)))
(is (= [] (:metrics structured-output)))
(is (= 2 (count (:errors structured-output))))))))))
+
+(deftest list-available-fields-tool-errors-test
+ (let [list-fields #(metadata-tools/list-available-fields-tool {:table_ids % :model_ids [] :metric_ids []})]
+ (testing "too many ids go back to the agent as output"
+ (is (= {:output "Too many table IDs provided (6). Limit to 5."}
+ (list-fields [1 2 3 4 5 6]))))
+ (testing "an unexpected error propagates to the agent loop"
+ (mt/with-dynamic-fn-redefs [metadata-tools/validate-id-count (fn [& _] (throw (ex-info "boom" {})))]
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"boom" (list-fields [])))))))
diff --git a/test/metabase/metabot/tools/resources_test.clj b/test/metabase/metabot/tools/resources_test.clj
index 83cbd70213d..33f59823a17 100644
--- a/test/metabase/metabot/tools/resources_test.clj
+++ b/test/metabase/metabot/tools/resources_test.clj
@@ -85,6 +85,15 @@
(is (thrown-with-msg? Exception #"Too many URIs"
(read-resource/read-resource {:uris uris}))))))
+(deftest read-resource-tool-errors-test
+ (testing "too many URIs go back to the agent as output"
+ (is (re-find #"^Too many URIs provided \(10\)"
+ (:output (read-resource/read-resource-tool {:uris (vec (repeat 10 "metabase://table/123"))})))))
+ (testing "an unexpected error propagates to the agent loop"
+ (mt/with-dynamic-fn-redefs [read-resource/read-resource (fn [_] (throw (ex-info "boom" {})))]
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"boom"
+ (read-resource/read-resource-tool {:uris ["metabase://databases"]}))))))
+
;; ===== Dispatch routing — every URI pattern routes to the expected handler =====
(def ^:private dispatch-cases
diff --git a/test/metabase/metabot/tools/save_entity_test.clj b/test/metabase/metabot/tools/save_entity_test.clj
index 76601e3f6a0..f3987fa2874 100644
--- a/test/metabase/metabot/tools/save_entity_test.clj
+++ b/test/metabase/metabot/tools/save_entity_test.clj
@@ -5,6 +5,7 @@
[metabase.lib.metadata :as lib.metadata]
[metabase.metabot.tools.save-entity :as save-entity]
[metabase.metabot.tools.shared :as shared]
+ [metabase.queries.core :as queries]
[metabase.test :as mt]
[metabase.util.json :as json]
[toucan2.core :as t2]))
@@ -221,3 +222,19 @@
:destination {:target_type "collection" :collection_id nil}}))]
(is (nil? (:data-parts result)))
(is (re-find #"No generated chart found" (:output result)))))))
+
+(deftest save-entity-errors-test
+ (testing "a dashboard that doesn't exist goes back to the agent as output"
+ (mt/with-current-user (mt/user->id :crowberto)
+ (is (= "Not found." (:output (save! {:target_type "dashboard" :dashboard_id Integer/MAX_VALUE}))))))
+ (testing "a collection the user can't write to goes back to the agent as output"
+ (mt/with-non-admin-groups-no-root-collection-perms
+ (mt/with-temp [:model/Collection coll {}]
+ (mt/with-current-user (mt/user->id :rasta)
+ (is (= "You don't have permissions to do that."
+ (:output (save! {:target_type "collection" :collection_id (:id coll)}))))))))
+ (testing "an unexpected error propagates to the agent loop"
+ (mt/with-dynamic-fn-redefs [queries/create-card! (fn [& _] (throw (ex-info "boom" {})))]
+ (mt/with-current-user (mt/user->id :crowberto)
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"boom"
+ (save! {:target_type "collection" :collection_id nil})))))))
diff --git a/test/metabase/metabot/tools/skills_test.clj b/test/metabase/metabot/tools/skills_test.clj
index b5d2f7816c5..daf52961ac2 100644
--- a/test/metabase/metabot/tools/skills_test.clj
+++ b/test/metabase/metabot/tools/skills_test.clj
@@ -2,7 +2,9 @@
(:require
[clojure.string :as str]
[clojure.test :refer [deftest is testing]]
- [metabase.metabot.tools.skills :as tools.skills]))
+ [metabase.metabot.skills :as skills]
+ [metabase.metabot.tools.skills :as tools.skills]
+ [metabase.test :as mt]))
(set! *warn-on-reflection* true)
@@ -31,3 +33,9 @@
(let [out (:output (tools.skills/load-skill-tool {:ids ["does-not-exist"]}))]
(is (str/includes? out "Unknown skill"))
(is (str/includes? out "does-not-exist")))))
+
+(deftest ^:parallel load-skill-unexpected-error-test
+ (testing "an unexpected error propagates to the agent loop"
+ (mt/with-dynamic-fn-redefs [skills/get-skill-by-id-string (fn [_] (throw (ex-info "boom" {})))]
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"boom"
+ (tools.skills/load-skill-tool {:ids ["construct-notebook-query-core"]}))))))
diff --git a/test/metabase/metabot/tools/sql_test.clj b/test/metabase/metabot/tools/sql_test.clj
index c5d9e88f6bd..decd3200c70 100644
--- a/test/metabase/metabot/tools/sql_test.clj
+++ b/test/metabase/metabot/tools/sql_test.clj
@@ -8,6 +8,8 @@
[metabase.metabot.tools.shared :as shared]
[metabase.metabot.tools.sql :as agent-sql]
[metabase.metabot.tools.sql.create :as create-sql-query-tools]
+ [metabase.metabot.tools.sql.edit :as edit-sql-query-tools]
+ [metabase.metabot.tools.sql.replace :as replace-sql-query-tools]
[metabase.permissions.core :as perms]
[metabase.permissions.models.permissions-group :as perms-group]
[metabase.test :as mt]))
@@ -286,3 +288,18 @@
:checklist "- [x] checked"
:new_query "SELECT 2"
:title "Results"})))))))))))
+
+(deftest edit-and-replace-sql-query-unexpected-error-test
+ (testing "edit_sql_query and replace_sql_query rethrow non-agent errors so they stay tracked as failures"
+ (mt/with-dynamic-fn-redefs [edit-sql-query-tools/edit-sql-query (fn [_] (throw (ex-info "boom" {})))
+ replace-sql-query-tools/replace-sql-query (fn [_] (throw (ex-info "boom" {})))]
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"boom"
+ (agent-sql/edit-sql-query-tool {:query_id "q-1"
+ :checklist "- [x] checked"
+ :edits [{:old_string "1" :new_string "2"}]
+ :title "Results"})))
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"boom"
+ (agent-sql/replace-sql-query-tool {:query_id "q-1"
+ :checklist "- [x] checked"
+ :new_query "SELECT 2"
+ :title "Results"}))))))
diff --git a/test/metabase/metabot/tools/subscriptions_test.clj b/test/metabase/metabot/tools/subscriptions_test.clj
index 205e71f675a..4690bb41223 100644
--- a/test/metabase/metabot/tools/subscriptions_test.clj
+++ b/test/metabase/metabot/tools/subscriptions_test.clj
@@ -3,6 +3,7 @@
(:require
[clojure.test :refer :all]
[metabase.channel.settings :as channel.settings]
+ [metabase.metabot.test-util :as test-util]
[metabase.metabot.tools.subscriptions :as agent-subscriptions]
[metabase.test :as mt]))
@@ -40,9 +41,9 @@
(let [result (agent-subscriptions/create-dashboard-subscription-tool
{:dashboard_id 0
- :schedule {:frequency "daily"}})]
+ :schedule {:frequency "daily" :hour 9}})]
;; dashboard_id 0 is int but won't match any dashboard
- (is (string? (:error result)))))))
+ (is (string? (:output result)))))))
(deftest create-dashboard-subscription-unknown-email-test
(testing "unknown email → 'no user with this email found'"
@@ -75,25 +76,29 @@
(deftest create-dashboard-subscription-schedule-keywords-test
(testing "schedule keywords are converted from snake_case to kebab-case"
- (let [captured-args (atom nil)]
+ (let [captured-args (atom nil)
+ sent-schedule (fn [schedule]
+ (agent-subscriptions/create-dashboard-subscription-tool
+ {:dashboard_id 1
+ :schedule schedule})
+ (:schedule @captured-args))]
(mt/with-dynamic-fn-redefs [agent-subscriptions/create-dashboard-subscription
(fn [args] (reset! captured-args args) {:output "success"})]
(mt/with-current-user (mt/user->id :crowberto)
- (agent-subscriptions/create-dashboard-subscription-tool
- {:dashboard_id 1
- :schedule {:frequency "weekly"
- :hour 9
- :day_of_week "monday"
- :day_of_month "first-mon"}}))
- (let [{:keys [schedule]} @captured-args]
- (is (= :weekly (:frequency schedule)))
- (is (= :monday (:day-of-week schedule)))
- (is (= :first-mon (:day-of-month schedule)))
- (is (= 9 (:hour schedule)))
- ;; snake_case keys should be absent
- (is (nil? (:day_of_week schedule)))
- (is (nil? (:day_of_month schedule))))))))
+ (is (= {:frequency :weekly :hour 9 :day-of-week :monday}
+ (sent-schedule {:frequency "weekly" :hour 9 :day_of_week "monday"})))
+ (is (= {:frequency :monthly :hour 9 :day-of-month :first-monday}
+ (sent-schedule {:frequency "monthly" :hour 9 :day_of_month "first-monday"}))))))))
+
+(deftest ^:parallel create-dashboard-subscription-unknown-day-of-week-test
+ (testing "a day_of_week outside the listed days is rejected at argument validation"
+ (is (=? #"Invalid tool arguments: `schedule` .*should be either \"sunday\".*"
+ (test-util/tool-boundary-error "create_dashboard_subscription"
+ #'agent-subscriptions/create-dashboard-subscription-tool
+ {:dashboard_id 1
+ :slack_channel "data-team"
+ :schedule {:frequency "weekly" :day_of_week "mo" :hour 9}})))))
;;; ---------------------------------------- Slack integration tests --------------------------------------------------
@@ -146,7 +151,7 @@
{:dashboard_id dash-id
:slack_channel ""
:schedule {:frequency "daily" :hour 9}})]
- (is (= {:error "slack_channel is required"} result))))))))
+ (is (= {:output "slack_channel is required"} result))))))))
(deftest create-dashboard-subscription-slack-not-configured-test
(testing "Slack not configured → error"
@@ -157,7 +162,7 @@
{:dashboard_id dash-id
:slack_channel "data-team"
:schedule {:frequency "daily" :hour 9}})]
- (is (= {:error "slack is not configured. Ask an admin to connect slack in Metabase settings."}
+ (is (= {:output "slack is not configured. Ask an admin to connect slack in Metabase settings."}
result))))))))
(deftest create-dashboard-subscription-slack-channel-not-found-test
@@ -175,4 +180,21 @@
{:dashboard_id dash-id
:slack_channel "no-such-channel"
:schedule {:frequency "daily" :hour 9}})]
- (is (= {:error "no slack channel found with this name"} result))))))))
+ (is (= {:output "no slack channel found with this name"} result))))))))
+
+(deftest create-dashboard-subscription-tool-errors-test
+ (let [subscribe! #(agent-subscriptions/create-dashboard-subscription-tool
+ {:dashboard_id %
+ :slack_channel "data-team"
+ :schedule {:frequency "daily" :hour 9}})]
+ (testing "a dashboard the user can't read goes back to the agent as output"
+ (mt/with-dynamic-fn-redefs [channel.settings/slack-configured? (constantly true)]
+ (mt/with-non-admin-groups-no-root-collection-perms
+ (mt/with-temp [:model/Dashboard {dash-id :id} {}]
+ (mt/with-current-user (mt/user->id :rasta)
+ (is (= "You don't have permissions to do that."
+ (:output (subscribe! dash-id)))))))))
+ (testing "an unexpected error propagates to the agent loop"
+ (mt/with-dynamic-fn-redefs [agent-subscriptions/create-dashboard-subscription
+ (fn [_] (throw (ex-info "boom" {})))]
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"boom" (subscribe! 1)))))))
diff --git a/test/metabase/metabot/tools/todo_test.clj b/test/metabase/metabot/tools/todo_test.clj
index b8d8807a0da..4ea3547271c 100644
--- a/test/metabase/metabot/tools/todo_test.clj
+++ b/test/metabase/metabot/tools/todo_test.clj
@@ -1,7 +1,9 @@
(ns metabase.metabot.tools.todo-test
(:require
[clojure.test :refer :all]
- [metabase.metabot.tools.todo :as todo]))
+ [metabase.metabot.tools.shared :as shared]
+ [metabase.metabot.tools.todo :as todo]
+ [metabase.test :as mt]))
(deftest ^:parallel todo-write-test
(testing "todo-write validates and stores todos in memory"
@@ -98,3 +100,14 @@
result (todo/todo-read {:memory-atom memory-atom})]
(is (contains? result :instructions))
(is (string? (:instructions result))))))
+
+(deftest ^:parallel todo-tools-errors-test
+ (testing "an invalid todo goes back to the agent as output"
+ (binding [shared/*memory-atom* (atom {:state {}})]
+ (is (= {:output "Todo item missing required 'id' field"}
+ (todo/todo-write-tool {:todos [{:id "" :content "Task" :status "pending" :priority "high"}]})))))
+ (testing "an unexpected error propagates to the agent loop"
+ (mt/with-dynamic-fn-redefs [todo/todo-write (fn [_] (throw (ex-info "boom" {})))
+ todo/todo-read (fn [_] (throw (ex-info "boom" {})))]
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"boom" (todo/todo-write-tool {:todos []})))
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"boom" (todo/todo-read-tool {}))))))
diff --git a/test/metabase/metabot/tools/util_test.clj b/test/metabase/metabot/tools/util_test.clj
index b8109eb6c6d..91498f3bacd 100644
--- a/test/metabase/metabot/tools/util_test.clj
+++ b/test/metabase/metabot/tools/util_test.clj
@@ -10,6 +10,20 @@
[metabase.permissions.models.permissions :as perms]
[metabase.test :as mt]))
+(deftest ^:parallel handle-agent-or-api-error-test
+ (testing "an API check's refusal goes back to the agent as output"
+ (doseq [status-code [400 403 404]]
+ (is (= {:output "Refused." :status-code status-code}
+ (metabot.tools.util/handle-agent-or-api-error (ex-info "Refused." {:status-code status-code}))))))
+ (testing "an agent error keeps its flags"
+ (is (= {:output "No access." :status-code 403 :terminal-error? true}
+ (metabot.tools.util/handle-agent-or-api-error
+ (ex-info "No access." {:agent-error? true :status-code 403 :terminal-error? true})))))
+ (testing "anything else is rethrown"
+ (doseq [data [{} {:status-code 500}]]
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"boom"
+ (metabot.tools.util/handle-agent-or-api-error (ex-info "boom" data)))))))
+
(deftest ^:parallel schedule->schedule-map-test
(testing "hourly schedule"
(is (= {:schedule_type "hourly"