The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Report unexpected Metabot tool failures as tool errors (#82969)
Andrei Simionescu5375ffed93c1
diff --git a/src/metabase/metabot/self/core.clj b/src/metabase/metabot/self/core.cljindex 0fc7493256b..07104eb35f0 100644--- a/src/metabase/metabot/self/core.clj+++ b/src/metabase/metabot/self/core.clj@@ -981,7 +981,7 @@(log/debugf "Tool %s: agent validation error: %s" safe-name (ex-message e)):else- (log/warn "Tool execution failed" {:tool-name safe-name :error (ex-message e)}))+ (log/error e "Tool execution failed" {:tool-name safe-name}))[{:type :tool-output-available:toolCallId tool-call-id:toolName tool-namediff --git a/src/metabase/metabot/tools/autogen_dashboard.clj b/src/metabase/metabot/tools/autogen_dashboard.cljindex d24e04c5a18..7d4461d1caa 100644--- a/src/metabase/metabot/tools/autogen_dashboard.clj+++ b/src/metabase/metabot/tools/autogen_dashboard.clj@@ -6,6 +6,7 @@[metabase.metabot.db :as metabot.db][metabase.metabot.scope :as scope][metabase.metabot.tools.shared :as shared]+ [metabase.metabot.tools.util :as metabot.tools.u][metabase.util.log :as log][metabase.util.malli :as mu]))@@ -153,7 +154,4 @@:instructions (:instructions result)}(seq data-parts) (assoc :data-parts data-parts)))(catch Exception e- (log/errorf "Error creating autogenerated dashboard: %s" (ex-message e))- (if (:agent-error? (ex-data e))- {:output (ex-message e)}- {:output (str "Failed to create autogenerated dashboard: " (or (ex-message e) "Unknown error"))}))))+ (metabot.tools.u/handle-agent-error e))))diff --git a/src/metabase/metabot/tools/charts.clj b/src/metabase/metabot/tools/charts.cljindex 432db464425..f09d9ccd7cd 100644--- a/src/metabase/metabot/tools/charts.clj+++ b/src/metabase/metabot/tools/charts.clj@@ -10,7 +10,7 @@[metabase.metabot.tools.shared :as shared][metabase.metabot.tools.shared.instructions :as instructions][metabase.metabot.tools.shared.llm-shape :as llm-shape]- [metabase.util.log :as log]+ [metabase.metabot.tools.util :as metabot.tools.u][metabase.util.malli :as mu]))(set! *warn-on-reflection* true)@@ -60,10 +60,7 @@:title title:description description})]})(catch Exception e- (log/errorf "Error creating chart: %s" (ex-message e))- (if (:agent-error? (ex-data e))- {:output (ex-message e)}- {:output (str "Failed to create chart: " (or (ex-message e) "Unknown error"))}))))+ (metabot.tools.u/handle-agent-error e))))(def ^:private edit-chart-schema[:map {:closed true}@@ -109,7 +106,4 @@:title title:description description})]})(catch Exception e- (log/errorf "Error editing chart: %s" (ex-message e))- (if (:agent-error? (ex-data e))- {:output (ex-message e)}- {:output (str "Failed to edit chart: " (or (ex-message e) "Unknown error"))}))))+ (metabot.tools.u/handle-agent-error e))))diff --git a/src/metabase/metabot/tools/clarification.clj b/src/metabase/metabot/tools/clarification.cljindex 278ee491f8f..f10a05d9c41 100644--- a/src/metabase/metabot/tools/clarification.clj+++ b/src/metabase/metabot/tools/clarification.clj@@ -45,9 +45,6 @@[{:keys [question options]} :- [:map {:closed true}[:question :string][:options {:optional true} [:maybe [:sequential :string]]]]]- (try- (let [result (ask-for-sql-clarification {:question question- :options options})]- (assoc result :output (format-clarification-output (:structured-output result))))- (catch Exception e- {:output (str "Failed to ask clarification: " (or (ex-message e) "Unknown error"))})))+ (let [result (ask-for-sql-clarification {:question question+ :options options})]+ (assoc result :output (format-clarification-output (:structured-output result)))))diff --git a/src/metabase/metabot/tools/create_alert.clj b/src/metabase/metabot/tools/create_alert.cljindex 2eaf152dc8a..4b16302b219 100644--- a/src/metabase/metabot/tools/create_alert.clj+++ b/src/metabase/metabot/tools/create_alert.clj@@ -11,7 +11,6 @@[metabase.metabot.tools.util :as metabot.tools.u][metabase.notification.api :as notification.api][metabase.util.cron :as u.cron]- [metabase.util.log :as log][metabase.util.malli :as mu]))(set! *warn-on-reflection* true)@@ -134,10 +133,10 @@`send_once` is optional (default false); when true the alert is deleted after it fires once."[{:keys [card_id send_condition schedule send_once]} :- alert-schema](let [slack-channel-id (:slack_channel_id (shared/current-context))]- (when-not slack-channel-id- (throw (ex-info "This tool can only be used from a Slack channel"- {:agent-error? true})))(try+ (when-not slack-channel-id+ (throw (ex-info "This tool can only be used from a Slack channel"+ {:agent-error? true})))(let [result (create-alert {:card-id card_id:send-condition (keyword send_condition):schedule schedule@@ -147,5 +146,4 @@{:output (:error result)}{:output (or (:output result) "Alert created successfully.")}))(catch Exception e- (log/errorf "Failed to create alert: %s" (ex-message e))- {:output (str "Failed to create alert: " (or (ex-message e) "Unknown error"))}))))+ (metabot.tools.u/handle-agent-or-api-error e)))))diff --git a/src/metabase/metabot/tools/create_dashboard_subscription.clj b/src/metabase/metabot/tools/create_dashboard_subscription.cljindex 4211e26576e..5d0b52f96be 100644--- a/src/metabase/metabot/tools/create_dashboard_subscription.clj+++ b/src/metabase/metabot/tools/create_dashboard_subscription.clj@@ -9,7 +9,6 @@[metabase.metabot.tools.shared :as shared][metabase.metabot.tools.util :as metabot.tools.u][metabase.pulse.api :as pulse.api]- [metabase.util.log :as log][metabase.util.malli :as mu][toucan2.core :as t2]))@@ -95,10 +94,10 @@[:dashboard_id :int][:schedule tools.create-alert/schedule-schema]]](let [slack-channel-id (:slack_channel_id (shared/current-context))]- (when-not slack-channel-id- (throw (ex-info "This tool can only be used from a Slack channel"- {:agent-error? true})))(try+ (when-not slack-channel-id+ (throw (ex-info "This tool can only be used from a Slack channel"+ {:agent-error? true})))(let [result (create-dashboard-subscription{:dashboard-id dashboard_id:schedule schedule@@ -107,5 +106,4 @@{:output (:error result)}{:output (or (:output result) "Dashboard subscription created successfully.")}))(catch Exception e- (log/errorf "Failed to create dashboard subscription: %s" (ex-message e))- {:output (str "Failed to create dashboard subscription: " (or (ex-message e) "Unknown error"))}))))+ (metabot.tools.u/handle-agent-or-api-error e)))))diff --git a/src/metabase/metabot/tools/document.clj b/src/metabase/metabot/tools/document.cljindex d313a86c70a..f9144339caa 100644--- a/src/metabase/metabot/tools/document.clj+++ b/src/metabase/metabot/tools/document.clj@@ -9,7 +9,6 @@[metabase.metabot.tools.sql.create :as create-sql-query-tools][metabase.metabot.tools.util :as metabot.tools.u][metabase.query-processor.core :as qp]- [metabase.util.log :as log][metabase.util.malli :as mu][metabase.warehouses.core :as warehouses]))@@ -124,8 +123,7 @@:structured-output {:database_id database-id:sql_engine (:engine db)}})))(catch Exception e- (log/errorf "Error collecting document schema: %s" (ex-message e))- {:output (str "Failed to collect schema: " (or (ex-message e) "Unknown error"))})))+ (metabot.tools.u/handle-agent-or-api-error e))))(def ^:private sql-chart-schema[:map {:closed true}@@ -175,10 +173,7 @@{:output "Draft chart payload generated from SQL query.":structured-output structured}))))(catch Exception e- (log/errorf "Error constructing SQL chart draft: %s" (ex-message e))- (if (:agent-error? (ex-data e))- (metabot.tools.u/handle-agent-error e)- {:output (str "Failed to construct SQL chart draft: " (or (ex-message e) "Unknown error"))}))))+ (metabot.tools.u/handle-agent-error e))))(def ^:private model-chart-schema"Schema for `document_construct_model_chart`. Mirrors `construct_notebook_query`'s@@ -222,7 +217,4 @@(or result{:output "Failed to construct model chart draft."})))(catch Exception e- (log/errorf "Error constructing model chart draft: %s" (ex-message e))- (if (:agent-error? (ex-data e))- (metabot.tools.u/handle-agent-error e)- {:output (str "Failed to construct model chart draft: " (or (ex-message e) "Unknown error"))}))))+ (metabot.tools.u/handle-agent-error e))))diff --git a/src/metabase/metabot/tools/metadata.clj b/src/metabase/metabot/tools/metadata.cljindex b587a391a03..d1ae1ee294f 100644--- a/src/metabase/metabot/tools/metadata.clj+++ b/src/metabase/metabot/tools/metadata.clj@@ -11,6 +11,7 @@[metabase.metabot.tools.shared :as shared][metabase.metabot.tools.shared.instructions :as instructions][metabase.metabot.tools.shared.llm-shape :as llm-shape]+ [metabase.metabot.tools.util :as metabot.tools.u][metabase.util.i18n :refer [tru]][metabase.util.log :as log][metabase.util.malli :as mu]))@@ -106,10 +107,7 @@:metrics metrics:errors errors}}))(catch Exception e- (log/errorf "Failed to fetch metadata: %s" (ex-message e))- (if (:agent-error? (ex-data e))- {:output (ex-message e)}- {:output (str "Failed to fetch metadata: " (or (ex-message e) "Unknown error"))}))))+ (metabot.tools.u/handle-agent-error e))))(defn- format-with-instructions[data instruction-text]diff --git a/src/metabase/metabot/tools/resources.clj b/src/metabase/metabot/tools/resources.cljindex 149eaf1eb12..ab5cdd4dc12 100644--- a/src/metabase/metabot/tools/resources.clj+++ b/src/metabase/metabot/tools/resources.clj@@ -81,6 +81,7 @@[metabase.metabot.tools.shared.content-store :as shared.content-store][metabase.metabot.tools.shared.instructions :as instructions][metabase.metabot.tools.shared.llm-shape :as llm-shape]+ [metabase.metabot.tools.util :as metabot.tools.u][metabase.models.interface :as mi][metabase.permissions.core :as perms][metabase.transforms.core :as transforms]@@ -1021,7 +1022,7 @@(str "Too many URIs provided (" (count uris) "). ""Please limit to " max-concurrent-uris " URIs maximum. ""Be more selective and focus on the most relevant items for the current task or fetch them in batches.")- {:uri-count (count uris) :max max-concurrent-uris})))+ {:agent-error? true :uri-count (count uris) :max max-concurrent-uris})));; Fetch all URIs (sequentially for now, could parallelize with pmap)(let [resources (mapv fetch-single-uri uris)@@ -1093,5 +1094,4 @@(try(read-resource {:uris uris})(catch Exception e- (log/errorf "Error in read_resource tool: %s" (ex-message e))- {:output (str "Failed to read resources: " (or (ex-message e) "Unknown error"))})))+ (metabot.tools.u/handle-agent-error e))))diff --git a/src/metabase/metabot/tools/save_entity.clj b/src/metabase/metabot/tools/save_entity.cljindex 7a62feab40c..64adbc49955 100644--- a/src/metabase/metabot/tools/save_entity.clj+++ b/src/metabase/metabot/tools/save_entity.clj@@ -18,10 +18,10 @@[metabase.metabot.scope :as scope][metabase.metabot.tmpl :as te][metabase.metabot.tools.shared :as shared]+ [metabase.metabot.tools.util :as metabot.tools.u][metabase.queries.core :as queries][metabase.query-permissions.core :as query-perms][metabase.util.i18n :refer [tru]]- [metabase.util.log :as log][metabase.util.malli :as mu][toucan2.core :as t2]))@@ -240,7 +240,4 @@;; as the settled "Saved …" step label:title (te/link question-name link)})]})(catch Exception e- (log/errorf "Error saving entity: %s" (ex-message e))- (if (:agent-error? (ex-data e))- {:output (ex-message e)}- {:output (str "Failed to save: " (or (ex-message e) "Unknown error"))}))))+ (metabot.tools.u/handle-agent-or-api-error e))))diff --git a/src/metabase/metabot/tools/skills.clj b/src/metabase/metabot/tools/skills.cljindex 763b7fdb5d0..ea342404787 100644--- a/src/metabase/metabot/tools/skills.clj+++ b/src/metabase/metabot/tools/skills.clj@@ -37,8 +37,4 @@[{:keys [ids]} :- [:map {:closed true}[:ids [:sequential[:string {:description "A skill id, e.g. construct-notebook-query-core"}]]]]]- (try- {:output (str/join "\n\n" (map load-one ids))}- (catch Exception e- (log/errorf "Failed to load skill(s): %s" (ex-message e))- {:output (str "Failed to load skill(s): " (or (ex-message e) "Unknown error"))})))+ {:output (str/join "\n\n" (map load-one ids))})diff --git a/src/metabase/metabot/tools/sql.clj b/src/metabase/metabot/tools/sql.cljindex a1593fc1e98..87093b991e0 100644--- a/src/metabase/metabot/tools/sql.clj+++ b/src/metabase/metabot/tools/sql.clj@@ -13,7 +13,6 @@[metabase.metabot.tools.sql.replace :as replace-sql-query-tools][metabase.metabot.tools.util :as metabot.tools.u][metabase.util :as u]- [metabase.util.log :as log][metabase.util.malli :as mu]))(set! *warn-on-reflection* true)@@ -187,10 +186,7 @@{:output (format-validation-error-output instr):instructions instr})))(catch Exception e- (log/errorf "Error editing SQL query: %s" (ex-message e))- (if (:agent-error? (ex-data e))- (metabot.tools.u/handle-agent-error e)- {:output (str "Failed to edit SQL query: " (or (ex-message e) "Unknown error"))}))))+ (metabot.tools.u/handle-agent-error e))));;; ──────────────────────────────────────────────────────────────────;;; Replace SQL query@@ -236,7 +232,4 @@{:output (format-validation-error-output instr):instructions instr})))(catch Exception e- (log/errorf "Error replacing SQL query: %s" (ex-message e))- (if (:agent-error? (ex-data e))- (metabot.tools.u/handle-agent-error e)- {:output (str "Failed to replace SQL query: " (or (ex-message e) "Unknown error"))}))))+ (metabot.tools.u/handle-agent-error e))))diff --git a/src/metabase/metabot/tools/subscriptions.clj b/src/metabase/metabot/tools/subscriptions.cljindex 16a9493f659..aefb8081142 100644--- a/src/metabase/metabot/tools/subscriptions.clj+++ b/src/metabase/metabot/tools/subscriptions.clj@@ -6,9 +6,9 @@[metabase.channel.settings :as channel.settings][metabase.metabot.db :as metabot.db][metabase.metabot.scope :as scope]+ [metabase.metabot.tools.create-alert :as tools.create-alert][metabase.metabot.tools.util :as metabot.tools.u][metabase.pulse.api :as pulse.api]- [metabase.util.log :as log][metabase.util.malli :as mu][toucan2.core :as t2]))@@ -85,11 +85,7 @@[:dashboard_id :int][:email {:optional true} [:maybe :string]][:slack_channel {:optional true} [:maybe :string]]- [:schedule [:map {:closed true}- [:frequency [:enum "hourly" "daily" "weekly" "monthly"]]- [:hour {:optional true} [:maybe :int]]- [:day_of_week {:optional true} [:maybe :string]]- [:day_of_month {:optional true} [:maybe :string]]]]])+ [:schedule tools.create-alert/schedule-schema]])(mu/defn ^{:tool-name "create_dashboard_subscription":scope scope/agent-dashboard-subscribe}@@ -105,19 +101,17 @@email address."[{:keys [dashboard_id email slack_channel schedule]} :- subscription-schema](try- (create-dashboard-subscription- {:dashboard-id dashboard_id- :email email- :slack-channel slack_channel- :schedule (-> schedule- (update :frequency keyword)- (cond->- (:day_of_week schedule) (-> (assoc :day-of-week (keyword (:day_of_week schedule)))- (dissoc :day_of_week))- (:day_of_month schedule) (-> (assoc :day-of-month (keyword (:day_of_month schedule)))- (dissoc :day_of_month))))})+ (-> (create-dashboard-subscription+ {:dashboard-id dashboard_id+ :email email+ :slack-channel slack_channel+ :schedule (-> schedule+ (update :frequency keyword)+ (cond->+ (:day_of_week schedule) (-> (assoc :day-of-week (keyword (:day_of_week schedule)))+ (dissoc :day_of_week))+ (:day_of_month schedule) (-> (assoc :day-of-month (keyword (:day_of_month schedule)))+ (dissoc :day_of_month))))})+ (set/rename-keys {:error :output}))(catch Exception e- (log/errorf "Error creating dashboard subscription: %s" (ex-message e))- (if (:agent-error? (ex-data e))- {:output (ex-message e)}- {:output (str "Failed to create dashboard subscription: " (or (ex-message e) "Unknown error"))}))))+ (metabot.tools.u/handle-agent-or-api-error e))))diff --git a/src/metabase/metabot/tools/todo.clj b/src/metabase/metabot/tools/todo.cljindex 41165bd0452..14728244d73 100644--- a/src/metabase/metabot/tools/todo.clj+++ b/src/metabase/metabot/tools/todo.clj@@ -7,6 +7,7 @@[metabase.metabot.agent.streaming :as streaming][metabase.metabot.scope :as scope][metabase.metabot.tools.shared :as shared]+ [metabase.metabot.tools.util :as metabot.tools.u][metabase.util.log :as log][metabase.util.malli :as mu]))@@ -128,9 +129,7 @@:memory-atom shared/*memory-atom*})format-todo-write-output)(catch Exception e- (if (:agent-error? (ex-data e))- {:output (ex-message e)}- {:output (str "Failed to update todo list: " (or (ex-message e) "Unknown error"))}))))+ (metabot.tools.u/handle-agent-error e))))(mu/defn ^{:tool-name "todo_read":scope scope/agent-todo-read}@@ -138,9 +137,6 @@"Read the current todo list from memory.Returns the list of todos that have been created during this conversation."[_args :- [:map {:closed true}]]- (try- (add-output- (todo-read {:memory-atom shared/*memory-atom*})- format-todo-read-output)- (catch Exception e- {:output (str "Failed to read todo list: " (or (ex-message e) "Unknown error"))})))+ (add-output+ (todo-read {:memory-atom shared/*memory-atom*})+ format-todo-read-output))diff --git a/src/metabase/metabot/tools/util.clj b/src/metabase/metabot/tools/util.cljindex 409d00a1069..8d2f96f0e26 100644--- a/src/metabase/metabot/tools/util.clj+++ b/src/metabase/metabot/tools/util.clj@@ -20,6 +20,15 @@terminal-error? (assoc :terminal-error? true))(throw e))))+(defn handle-agent-or-api-error+ "Return an agent output for agent errors and API check refusals, re-throw `e` otherwise. A refusal is an exception+ with a 4xx `:status-code`, like the 403 or 404 from [[api/read-check]]."+ [e]+ (let [{:keys [agent-error? status-code]} (ex-data e)]+ (if (and (not agent-error?) (int? status-code) (<= 400 status-code 499))+ {:output (ex-message e) :status-code status-code}+ (handle-agent-error e))))+(defn convert-field-type"Return tool type for `column`."[column]diff --git a/test/metabase/metabot/self_test.clj b/test/metabase/metabot/self_test.cljindex 12832d18d3c..7303d84e8dc 100644--- a/test/metabase/metabot/self_test.clj+++ b/test/metabase/metabot/self_test.clj@@ -592,6 +592,21 @@:type string?}}(last result))))))+(deftest ^:parallel tool-failure-log-level-test+ (let [run-failing (fn [e]+ (into [] (self.core/tool-executor-xf {"failing" {:fn (fn [_] (throw e))}})+ (test-util/parts->aisdk-chunks+ [{:type :tool-input :id "call-f" :function "failing" :arguments {}}])))]+ (testing "an unexpected tool failure is logged at error with its exception"+ (let [e (ex-info "boom" {})]+ (log.capture/with-log-messages-for-level [messages [metabase.metabot.self.core :error]]+ (run-failing e)+ (is (= [[:error e]] (map (juxt :level :e) (messages)))))))+ (testing "an agent error is not"+ (log.capture/with-log-messages-for-level [messages [metabase.metabot.self.core :error]]+ (run-failing (ex-info "No such field" {:agent-error? true}))+ (is (empty? (messages)))))))+(deftest ^:parallel tool-executor-xf-test-6(testing "tool-executor-xf handles nil arguments for no-arg tools"(let [chunks (test-util/parts->aisdk-chunksdiff --git a/test/metabase/metabot/tools/autogen_dashboard_test.clj b/test/metabase/metabot/tools/autogen_dashboard_test.cljindex 3b7c87c5f23..b82803a7233 100644--- a/test/metabase/metabot/tools/autogen_dashboard_test.clj+++ b/test/metabase/metabot/tools/autogen_dashboard_test.clj@@ -132,3 +132,12 @@(autogen-dashboard/create-autogenerated-dashboard{:source {:report_id 999999}:memory-atom (atom {:state {}})})))))++(deftest create-autogenerated-dashboard-tool-errors-test+ (testing "a query missing from memory goes back to the agent as output"+ (is (= {:output "Query not found in memory with ID: q-9. Available queries: []"}+ (autogen-dashboard/create-autogenerated-dashboard-tool {:source {:query_id "q-9"}}))))+ (testing "an unexpected error propagates to the agent loop"+ (mt/with-dynamic-fn-redefs [autogen-dashboard/create-autogenerated-dashboard (fn [_] (throw (ex-info "boom" {})))]+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"boom"+ (autogen-dashboard/create-autogenerated-dashboard-tool {:source {:table_id 1}}))))))diff --git a/test/metabase/metabot/tools/charts_test.clj b/test/metabase/metabot/tools/charts_test.cljindex f5050dc27aa..b3721882b38 100644--- a/test/metabase/metabot/tools/charts_test.clj+++ b/test/metabase/metabot/tools/charts_test.clj@@ -5,7 +5,10 @@(:require[clojure.test :refer :all][metabase.metabot.tools.charts :as charts]- [metabase.metabot.tools.shared :as shared]))+ [metabase.metabot.tools.charts.create :as create-chart-tools]+ [metabase.metabot.tools.charts.edit :as edit-chart-tools]+ [metabase.metabot.tools.shared :as shared]+ [metabase.test :as mt]));; create-chart only needs the query present in queries-state; the link builder;; json-encodes it and `->legacy-mbql` passes a non-MBQL 5 value through unchanged,@@ -75,3 +78,21 @@;; only looked at the chart's own (here empty) :queries, landing on nil.(is (= {:query-id "q-1" :query stub-query}(select-keys (:structured-output result) [:query-id :query])))))))++(deftest chart-tools-errors-test+ (testing "a missing query goes back to the agent as output"+ (is (re-find #"^Query not found with ID: q-9"+ (:output (binding [shared/*memory-atom* (atom {:state {:queries {}}})]+ (charts/create-chart-tool {:data_source {:query_id "q-9"}+ :viz_settings {:chart_type "bar"}+ :title "Orders by month"+ :description "Monthly count of orders."}))))))+ (testing "an unexpected error propagates to the agent loop"+ (mt/with-dynamic-fn-redefs [create-chart-tools/create-chart (fn [_] (throw (ex-info "boom" {})))+ edit-chart-tools/edit-chart (fn [_] (throw (ex-info "boom" {})))]+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"boom" (run-create-chart)))+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"boom"+ (charts/edit-chart-tool {:chart_id "c-1"+ :new_viz_settings {:chart_type "pie"}+ :title "Orders by month"+ :description "Monthly count of orders."}))))))diff --git a/test/metabase/metabot/tools/clarification_test.clj b/test/metabase/metabot/tools/clarification_test.cljindex 13438c75def..98b188212de 100644--- a/test/metabase/metabot/tools/clarification_test.clj+++ b/test/metabase/metabot/tools/clarification_test.clj@@ -2,7 +2,8 @@(:require[clojure.string :as str][clojure.test :refer :all]- [metabase.metabot.tools.clarification :as ask-clarification]))+ [metabase.metabot.tools.clarification :as ask-clarification]+ [metabase.test :as mt]))(deftest ask-for-sql-clarification-test(testing "returns structured-output (the success signal that lets a profile end the turn)"@@ -36,3 +37,9 @@"ask_for_sql_clarification must return :structured-output so the agent loop can stop")(is (not (contains? result :final-response?))"the tool-level :final-response? flag has been removed in favour of :terminal-tools"))))++(deftest ask-for-sql-clarification-tool-unexpected-error-test+ (testing "an unexpected error propagates to the agent loop"+ (mt/with-dynamic-fn-redefs [ask-clarification/ask-for-sql-clarification (fn [_] (throw (ex-info "boom" {})))]+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"boom"+ (ask-clarification/ask-for-sql-clarification-tool {:question "Which table?"}))))))diff --git a/test/metabase/metabot/tools/create_alert_test.clj b/test/metabase/metabot/tools/create_alert_test.cljindex 93c5862b418..d21791f1ab3 100644--- a/test/metabase/metabot/tools/create_alert_test.clj+++ b/test/metabase/metabot/tools/create_alert_test.clj@@ -4,6 +4,7 @@[metabase.channel.settings :as channel.settings][metabase.metabot.tools.create-alert:as metabot.tools.create-alert]+ [metabase.metabot.tools.shared :as shared][metabase.test :as mt]))(defn- alert-data [card-id]@@ -74,3 +75,24 @@channel.settings/slack-cached-channels-and-usernames (constantly {:channels []})](is (= {:error "no slack channel found with this name"}(invoke-tool (assoc base-data :slack-channel "no-such-channel")))))))))++(deftest create-alert-tool-errors-test+ (let [create-alert! #(binding [shared/*memory-atom* (atom {:context {:slack_channel_id "C123"}})]+ (metabot.tools.create-alert/create-alert-tool+ {:card_id %+ :send_condition "has_result"+ :schedule {:frequency "daily" :hour 9}}))]+ (testing "a call from outside Slack goes back to the agent as output"+ (is (= {:output "This tool can only be used from a Slack channel"}+ (metabot.tools.create-alert/create-alert-tool+ {:card_id 1 :send_condition "has_result" :schedule {:frequency "daily" :hour 9}}))))+ (testing "a card the user can't read goes back to the agent as output"+ (mt/with-dynamic-fn-redefs [channel.settings/slack-configured? (constantly true)]+ (mt/with-non-admin-groups-no-root-collection-perms+ (mt/with-temp [:model/Card {card-id :id} {}]+ (mt/with-current-user (mt/user->id :rasta)+ (is (= "You don't have permissions to do that."+ (:output (create-alert! card-id)))))))))+ (testing "an unexpected error propagates to the agent loop"+ (mt/with-dynamic-fn-redefs [metabot.tools.create-alert/create-alert (fn [_] (throw (ex-info "boom" {})))]+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"boom" (create-alert! 1)))))))diff --git a/test/metabase/metabot/tools/create_dashboard_subscription_test.clj b/test/metabase/metabot/tools/create_dashboard_subscription_test.cljnew file mode 100644index 00000000000..00f69c375a2--- /dev/null+++ b/test/metabase/metabot/tools/create_dashboard_subscription_test.clj@@ -0,0 +1,10 @@+(ns metabase.metabot.tools.create-dashboard-subscription-test+ (:require+ [clojure.test :refer :all]+ [metabase.metabot.tools.create-dashboard-subscription :as tools.create-dashboard-subscription]))++(deftest ^:parallel slackbot-create-dashboard-subscription-outside-slack-test+ (testing "a call from outside Slack goes back to the agent as output"+ (is (= {:output "This tool can only be used from a Slack channel"}+ (tools.create-dashboard-subscription/slackbot-create-dashboard-subscription-tool+ {:dashboard_id 1 :schedule {:frequency "daily" :hour 9}})))))diff --git a/test/metabase/metabot/tools/document_test.clj b/test/metabase/metabot/tools/document_test.cljindex a735b4f58aa..f1e9f42476a 100644--- a/test/metabase/metabot/tools/document_test.clj+++ b/test/metabase/metabot/tools/document_test.clj@@ -160,3 +160,31 @@structured (:structured-output result)](is (= chart-type (:display structured)))(is (= chart-type (:chart_type structured))))))))++(deftest document-tools-errors-test+ (testing "a database that no longer exists goes back to the agent as output"+ (mt/with-dynamic-fn-redefs [shared/current-context (fn [] {:references {"database:999999" "Gone"}})]+ (mt/with-current-user (mt/user->id :crowberto)+ (is (= "Not found." (:output (document-tools/document-schema-collect-tool {})))))))+ (testing "an unexpected error propagates to the agent loop"+ (mt/with-dynamic-fn-redefs [shared/current-context (fn [] {:references {"database:1" "Test Database"}})+ warehouses/get-database (fn [_] (throw (ex-info "boom" {})))+ create-sql-query-tools/create-sql-query (fn [_] (throw (ex-info "boom" {})))+ construct-tools/construct-notebook-query-tool (fn [_] (throw (ex-info "boom" {})))]+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"boom"+ (document-tools/document-schema-collect-tool {})))+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"boom"+ (document-tools/document-construct-sql-chart-tool+ {:database_id 1+ :name "Test Name"+ :description "Test Desc"+ :analysis "Test Analysis"+ :approach "Test Approach"+ :sql "SELECT 1"+ :viz_settings {:chart_type "bar"}})))+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"boom"+ (document-tools/document-construct-model-chart-tool+ {:name "Test Name"+ :description "Test Desc"+ :query ""+ :viz_settings {:chart_type "bar"}}))))))diff --git a/test/metabase/metabot/tools/metadata_test.clj b/test/metabase/metabot/tools/metadata_test.cljindex 6bafd8cd24a..79a1bb521ea 100644--- a/test/metabase/metabot/tools/metadata_test.clj+++ b/test/metabase/metabot/tools/metadata_test.clj@@ -102,3 +102,12 @@(is (= [] (:models structured-output)))(is (= [] (:metrics structured-output)))(is (= 2 (count (:errors structured-output))))))))))++(deftest list-available-fields-tool-errors-test+ (let [list-fields #(metadata-tools/list-available-fields-tool {:table_ids % :model_ids [] :metric_ids []})]+ (testing "too many ids go back to the agent as output"+ (is (= {:output "Too many table IDs provided (6). Limit to 5."}+ (list-fields [1 2 3 4 5 6]))))+ (testing "an unexpected error propagates to the agent loop"+ (mt/with-dynamic-fn-redefs [metadata-tools/validate-id-count (fn [& _] (throw (ex-info "boom" {})))]+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"boom" (list-fields [])))))))diff --git a/test/metabase/metabot/tools/resources_test.clj b/test/metabase/metabot/tools/resources_test.cljindex 83cbd70213d..33f59823a17 100644--- a/test/metabase/metabot/tools/resources_test.clj+++ b/test/metabase/metabot/tools/resources_test.clj@@ -85,6 +85,15 @@(is (thrown-with-msg? Exception #"Too many URIs"(read-resource/read-resource {:uris uris}))))))+(deftest read-resource-tool-errors-test+ (testing "too many URIs go back to the agent as output"+ (is (re-find #"^Too many URIs provided \(10\)"+ (:output (read-resource/read-resource-tool {:uris (vec (repeat 10 "metabase://table/123"))})))))+ (testing "an unexpected error propagates to the agent loop"+ (mt/with-dynamic-fn-redefs [read-resource/read-resource (fn [_] (throw (ex-info "boom" {})))]+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"boom"+ (read-resource/read-resource-tool {:uris ["metabase://databases"]}))))))+;; ===== Dispatch routing — every URI pattern routes to the expected handler =====(def ^:private dispatch-casesdiff --git a/test/metabase/metabot/tools/save_entity_test.clj b/test/metabase/metabot/tools/save_entity_test.cljindex 76601e3f6a0..f3987fa2874 100644--- a/test/metabase/metabot/tools/save_entity_test.clj+++ b/test/metabase/metabot/tools/save_entity_test.clj@@ -5,6 +5,7 @@[metabase.lib.metadata :as lib.metadata][metabase.metabot.tools.save-entity :as save-entity][metabase.metabot.tools.shared :as shared]+ [metabase.queries.core :as queries][metabase.test :as mt][metabase.util.json :as json][toucan2.core :as t2]))@@ -221,3 +222,19 @@:destination {:target_type "collection" :collection_id nil}}))](is (nil? (:data-parts result)))(is (re-find #"No generated chart found" (:output result)))))))++(deftest save-entity-errors-test+ (testing "a dashboard that doesn't exist goes back to the agent as output"+ (mt/with-current-user (mt/user->id :crowberto)+ (is (= "Not found." (:output (save! {:target_type "dashboard" :dashboard_id Integer/MAX_VALUE}))))))+ (testing "a collection the user can't write to goes back to the agent as output"+ (mt/with-non-admin-groups-no-root-collection-perms+ (mt/with-temp [:model/Collection coll {}]+ (mt/with-current-user (mt/user->id :rasta)+ (is (= "You don't have permissions to do that."+ (:output (save! {:target_type "collection" :collection_id (:id coll)}))))))))+ (testing "an unexpected error propagates to the agent loop"+ (mt/with-dynamic-fn-redefs [queries/create-card! (fn [& _] (throw (ex-info "boom" {})))]+ (mt/with-current-user (mt/user->id :crowberto)+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"boom"+ (save! {:target_type "collection" :collection_id nil})))))))diff --git a/test/metabase/metabot/tools/skills_test.clj b/test/metabase/metabot/tools/skills_test.cljindex b5d2f7816c5..daf52961ac2 100644--- a/test/metabase/metabot/tools/skills_test.clj+++ b/test/metabase/metabot/tools/skills_test.clj@@ -2,7 +2,9 @@(:require[clojure.string :as str][clojure.test :refer [deftest is testing]]- [metabase.metabot.tools.skills :as tools.skills]))+ [metabase.metabot.skills :as skills]+ [metabase.metabot.tools.skills :as tools.skills]+ [metabase.test :as mt]))(set! *warn-on-reflection* true)@@ -31,3 +33,9 @@(let [out (:output (tools.skills/load-skill-tool {:ids ["does-not-exist"]}))](is (str/includes? out "Unknown skill"))(is (str/includes? out "does-not-exist")))))++(deftest ^:parallel load-skill-unexpected-error-test+ (testing "an unexpected error propagates to the agent loop"+ (mt/with-dynamic-fn-redefs [skills/get-skill-by-id-string (fn [_] (throw (ex-info "boom" {})))]+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"boom"+ (tools.skills/load-skill-tool {:ids ["construct-notebook-query-core"]}))))))diff --git a/test/metabase/metabot/tools/sql_test.clj b/test/metabase/metabot/tools/sql_test.cljindex c5d9e88f6bd..decd3200c70 100644--- a/test/metabase/metabot/tools/sql_test.clj+++ b/test/metabase/metabot/tools/sql_test.clj@@ -8,6 +8,8 @@[metabase.metabot.tools.shared :as shared][metabase.metabot.tools.sql :as agent-sql][metabase.metabot.tools.sql.create :as create-sql-query-tools]+ [metabase.metabot.tools.sql.edit :as edit-sql-query-tools]+ [metabase.metabot.tools.sql.replace :as replace-sql-query-tools][metabase.permissions.core :as perms][metabase.permissions.models.permissions-group :as perms-group][metabase.test :as mt]))@@ -286,3 +288,18 @@:checklist "- [x] checked":new_query "SELECT 2":title "Results"})))))))))))++(deftest edit-and-replace-sql-query-unexpected-error-test+ (testing "edit_sql_query and replace_sql_query rethrow non-agent errors so they stay tracked as failures"+ (mt/with-dynamic-fn-redefs [edit-sql-query-tools/edit-sql-query (fn [_] (throw (ex-info "boom" {})))+ replace-sql-query-tools/replace-sql-query (fn [_] (throw (ex-info "boom" {})))]+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"boom"+ (agent-sql/edit-sql-query-tool {:query_id "q-1"+ :checklist "- [x] checked"+ :edits [{:old_string "1" :new_string "2"}]+ :title "Results"})))+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"boom"+ (agent-sql/replace-sql-query-tool {:query_id "q-1"+ :checklist "- [x] checked"+ :new_query "SELECT 2"+ :title "Results"}))))))diff --git a/test/metabase/metabot/tools/subscriptions_test.clj b/test/metabase/metabot/tools/subscriptions_test.cljindex 205e71f675a..4690bb41223 100644--- a/test/metabase/metabot/tools/subscriptions_test.clj+++ b/test/metabase/metabot/tools/subscriptions_test.clj@@ -3,6 +3,7 @@(:require[clojure.test :refer :all][metabase.channel.settings :as channel.settings]+ [metabase.metabot.test-util :as test-util][metabase.metabot.tools.subscriptions :as agent-subscriptions][metabase.test :as mt]))@@ -40,9 +41,9 @@(let [result (agent-subscriptions/create-dashboard-subscription-tool{:dashboard_id 0:email "[email protected]"- :schedule {:frequency "daily"}})]+ :schedule {:frequency "daily" :hour 9}})];; dashboard_id 0 is int but won't match any dashboard- (is (string? (:error result)))))))+ (is (string? (:output result)))))))(deftest create-dashboard-subscription-unknown-email-test(testing "unknown email → 'no user with this email found'"@@ -75,25 +76,29 @@(deftest create-dashboard-subscription-schedule-keywords-test(testing "schedule keywords are converted from snake_case to kebab-case"- (let [captured-args (atom nil)]+ (let [captured-args (atom nil)+ sent-schedule (fn [schedule]+ (agent-subscriptions/create-dashboard-subscription-tool+ {:dashboard_id 1+ :email "[email protected]"+ :schedule schedule})+ (:schedule @captured-args))](mt/with-dynamic-fn-redefs [agent-subscriptions/create-dashboard-subscription(fn [args] (reset! captured-args args) {:output "success"})](mt/with-current-user (mt/user->id :crowberto)- (agent-subscriptions/create-dashboard-subscription-tool- {:dashboard_id 1- :email "[email protected]"- :schedule {:frequency "weekly"- :hour 9- :day_of_week "monday"- :day_of_month "first-mon"}}))- (let [{:keys [schedule]} @captured-args]- (is (= :weekly (:frequency schedule)))- (is (= :monday (:day-of-week schedule)))- (is (= :first-mon (:day-of-month schedule)))- (is (= 9 (:hour schedule)))- ;; snake_case keys should be absent- (is (nil? (:day_of_week schedule)))- (is (nil? (:day_of_month schedule))))))))+ (is (= {:frequency :weekly :hour 9 :day-of-week :monday}+ (sent-schedule {:frequency "weekly" :hour 9 :day_of_week "monday"})))+ (is (= {:frequency :monthly :hour 9 :day-of-month :first-monday}+ (sent-schedule {:frequency "monthly" :hour 9 :day_of_month "first-monday"}))))))))++(deftest ^:parallel create-dashboard-subscription-unknown-day-of-week-test+ (testing "a day_of_week outside the listed days is rejected at argument validation"+ (is (=? #"Invalid tool arguments: `schedule` .*should be either \"sunday\".*"+ (test-util/tool-boundary-error "create_dashboard_subscription"+ #'agent-subscriptions/create-dashboard-subscription-tool+ {:dashboard_id 1+ :slack_channel "data-team"+ :schedule {:frequency "weekly" :day_of_week "mo" :hour 9}})))));;; ---------------------------------------- Slack integration tests --------------------------------------------------@@ -146,7 +151,7 @@{:dashboard_id dash-id:slack_channel "":schedule {:frequency "daily" :hour 9}})]- (is (= {:error "slack_channel is required"} result))))))))+ (is (= {:output "slack_channel is required"} result))))))))(deftest create-dashboard-subscription-slack-not-configured-test(testing "Slack not configured → error"@@ -157,7 +162,7 @@{:dashboard_id dash-id:slack_channel "data-team":schedule {:frequency "daily" :hour 9}})]- (is (= {:error "slack is not configured. Ask an admin to connect slack in Metabase settings."}+ (is (= {:output "slack is not configured. Ask an admin to connect slack in Metabase settings."}result))))))))(deftest create-dashboard-subscription-slack-channel-not-found-test@@ -175,4 +180,21 @@{:dashboard_id dash-id:slack_channel "no-such-channel":schedule {:frequency "daily" :hour 9}})]- (is (= {:error "no slack channel found with this name"} result))))))))+ (is (= {:output "no slack channel found with this name"} result))))))))++(deftest create-dashboard-subscription-tool-errors-test+ (let [subscribe! #(agent-subscriptions/create-dashboard-subscription-tool+ {:dashboard_id %+ :slack_channel "data-team"+ :schedule {:frequency "daily" :hour 9}})]+ (testing "a dashboard the user can't read goes back to the agent as output"+ (mt/with-dynamic-fn-redefs [channel.settings/slack-configured? (constantly true)]+ (mt/with-non-admin-groups-no-root-collection-perms+ (mt/with-temp [:model/Dashboard {dash-id :id} {}]+ (mt/with-current-user (mt/user->id :rasta)+ (is (= "You don't have permissions to do that."+ (:output (subscribe! dash-id)))))))))+ (testing "an unexpected error propagates to the agent loop"+ (mt/with-dynamic-fn-redefs [agent-subscriptions/create-dashboard-subscription+ (fn [_] (throw (ex-info "boom" {})))]+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"boom" (subscribe! 1)))))))diff --git a/test/metabase/metabot/tools/todo_test.clj b/test/metabase/metabot/tools/todo_test.cljindex b8d8807a0da..4ea3547271c 100644--- a/test/metabase/metabot/tools/todo_test.clj+++ b/test/metabase/metabot/tools/todo_test.clj@@ -1,7 +1,9 @@(ns metabase.metabot.tools.todo-test(:require[clojure.test :refer :all]- [metabase.metabot.tools.todo :as todo]))+ [metabase.metabot.tools.shared :as shared]+ [metabase.metabot.tools.todo :as todo]+ [metabase.test :as mt]))(deftest ^:parallel todo-write-test(testing "todo-write validates and stores todos in memory"@@ -98,3 +100,14 @@result (todo/todo-read {:memory-atom memory-atom})](is (contains? result :instructions))(is (string? (:instructions result))))))++(deftest ^:parallel todo-tools-errors-test+ (testing "an invalid todo goes back to the agent as output"+ (binding [shared/*memory-atom* (atom {:state {}})]+ (is (= {:output "Todo item missing required 'id' field"}+ (todo/todo-write-tool {:todos [{:id "" :content "Task" :status "pending" :priority "high"}]})))))+ (testing "an unexpected error propagates to the agent loop"+ (mt/with-dynamic-fn-redefs [todo/todo-write (fn [_] (throw (ex-info "boom" {})))+ todo/todo-read (fn [_] (throw (ex-info "boom" {})))]+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"boom" (todo/todo-write-tool {:todos []})))+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"boom" (todo/todo-read-tool {}))))))diff --git a/test/metabase/metabot/tools/util_test.clj b/test/metabase/metabot/tools/util_test.cljindex b8109eb6c6d..91498f3bacd 100644--- a/test/metabase/metabot/tools/util_test.clj+++ b/test/metabase/metabot/tools/util_test.clj@@ -10,6 +10,20 @@[metabase.permissions.models.permissions :as perms][metabase.test :as mt]))+(deftest ^:parallel handle-agent-or-api-error-test+ (testing "an API check's refusal goes back to the agent as output"+ (doseq [status-code [400 403 404]]+ (is (= {:output "Refused." :status-code status-code}+ (metabot.tools.util/handle-agent-or-api-error (ex-info "Refused." {:status-code status-code}))))))+ (testing "an agent error keeps its flags"+ (is (= {:output "No access." :status-code 403 :terminal-error? true}+ (metabot.tools.util/handle-agent-or-api-error+ (ex-info "No access." {:agent-error? true :status-code 403 :terminal-error? true})))))+ (testing "anything else is rethrown"+ (doseq [data [{} {:status-code 500}]]+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"boom"+ (metabot.tools.util/handle-agent-or-api-error (ex-info "boom" data)))))))+(deftest ^:parallel schedule->schedule-map-test(testing "hourly schedule"(is (= {:schedule_type "hourly"