Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Let Metabot recover once from an undeclared tool error (#86032)

Andrei Simionescu2ecdce58fcbf
diff --git a/src/metabase/metabot/agent/core.clj b/src/metabase/metabot/agent/core.clj
index f85947d9113..ec7b4efd9c9 100644
--- a/src/metabase/metabot/agent/core.clj
+++ b/src/metabase/metabot/agent/core.clj
@@ -25,6 +25,7 @@
[metabase.metabot.self :as self]
[metabase.metabot.self.schema :as self.schema]
[metabase.metabot.tools :as tools]
+ [metabase.metabot.tools.runtime :as tools.runtime]
[metabase.util :as u]
[metabase.util.async :as async.u]
[metabase.util.i18n :refer [tru]]
@@ -219,19 +220,41 @@
[parts]
(some #(= (:type %) :error) parts))
-(defn- terminal-error-message
- "The user-facing message from a tool failure no retry can fix, or nil if there was none.
+(def ^:private max-undeclared-failures
+ "How many ordinary exceptions that no tool declared a turn survives when its model can answer in text. The next one
+ ends it, which stops a retry loop."
+ 1)
- An unrecoverable failure is one only the user can act on — a permission denial, a server fault —
- so the turn ends. The message is the `:user-message` the tool chose; a tool that set none gets the
- generic one, because `:output` is written for the model and says only that the call failed."
- [parts]
- (some (fn [part]
- (when (and (= (:type part) :tool-output)
- (= :unrecoverable (get-in part [:result :error :class])))
- (or (not-empty (get-in part [:result :error :user-message]))
- (tru "Something went wrong."))))
- parts))
+(defn- undeclared-failure?
+ "Whether `part` is a tool call that failed with an ordinary exception no tool declared. A tool that writes reports
+ one as `:unconfirmed-write` instead, which ends the turn (see `tools.runtime/invoke`)."
+ [part]
+ (and (= (:type part) :tool-output)
+ (tools.runtime/undeclared? (get-in part [:result :error :code]))))
+
+(defn- terminal-error-message
+ "The user-facing message that ends the turn after this iteration's tool calls, or nil if the turn goes on.
+
+ An unrecoverable failure ends the turn: one only the user can act on, such as a permission denial
+ or a failed write that may have been saved, or one the model can't work around, such as a JVM
+ error. The message is the error's `:user-message`; an error without one gets the generic one,
+ because `:output` is written for the model and says only that the call failed.
+
+ An ordinary exception that no tool declared ([[undeclared-failure?]]) ends the turn only when
+ `undeclared-failures`, the turn's count of them with this iteration's included, passes
+ [[max-undeclared-failures]]. In a `profile` that requires a tool call the first one ends it,
+ because that model can't answer in text after it. The message is then the generic one."
+ [profile parts undeclared-failures]
+ (let [terminal (some (fn [part]
+ (when (and (= (:type part) :tool-output)
+ (= :unrecoverable (get-in part [:result :error :class]))
+ (not (undeclared-failure? part)))
+ (get-in part [:result :error])))
+ parts)
+ allowed (if (:required-tool-call? profile) 0 max-undeclared-failures)]
+ (when (or terminal (< allowed undeclared-failures))
+ (or (not-empty (:user-message terminal))
+ (tru "Something went wrong.")))))
(defn- should-continue?
"Determine if agent should continue iterating."
@@ -561,16 +584,30 @@
:permission/metabot)}
tracking-opts)}))
+(defn- stop-recording-rf
+ "Wrap `rf` so that `stopped?` turns true once `rf` returns `reduced`.
+
+ The loop can't learn that its consumer stopped from what an iteration returns: the eductions an iteration reduces
+ unwrap a `reduced` before they return."
+ [rf stopped?]
+ (fn [acc part]
+ (u/prog1 (rf acc part)
+ (when (reduced? <>)
+ (vreset! stopped? true)))))
+
(defn- initial-loop-state
"Create initial loop state from agent config and reduction context."
[agent rf init usage-atom]
- {:agent agent
- :rf rf
- :result init
- :iteration 1
- :status :continue
- :usage-atom usage-atom
- :sent-parts []})
+ (let [consumer-stopped? (volatile! false)]
+ {:agent agent
+ :rf (stop-recording-rf rf consumer-stopped?)
+ :consumer-stopped? consumer-stopped?
+ :result init
+ :iteration 1
+ :status :continue
+ :usage-atom usage-atom
+ :sent-parts []
+ :undeclared-failures 0}))
(defn- final-state-part [memory]
{:type :data, :data-type "state", :version 1, :data (memory/get-state memory)})
@@ -612,7 +649,7 @@
Streams parts to the consumer as they arrive while simultaneously accumulating
them for memory updates and control flow decisions."
- [{:keys [agent rf result iteration usage-atom sent-parts] :as loop-state}]
+ [{:keys [agent rf result iteration usage-atom sent-parts undeclared-failures consumer-stopped?] :as loop-state}]
(with-span :debug {:name :metabot.agent/loop-step
:iteration iteration}
(let [{:keys [profile tools system-msg context memory-atom tracking-opts]} agent
@@ -659,15 +696,16 @@
(do
(log/debug "Got parts" {:count (count parts) :types (mapv :type parts)})
(swap! memory-atom update-memory parts)
- ;; Every profile, not just the `:required-tool-call?` ones. An unrecoverable failure is
- ;; defined as one only the user can act on, so the class is the decision and the profile
- ;; is not part of it. A tool that wants the agent to carry on past a failure declares it
+ ;; Every profile, not just the `:required-tool-call?` ones. The failure decides, and for an
+ ;; ordinary exception that no tool declared, so do the profile and the turn's count of them.
+ ;; A tool that wants the agent to carry on past a failure every time declares it
;; recoverable, which is a choice made at the error, where the reason for it is visible.
- (let [terminal-error (terminal-error-message parts)]
+ (let [undeclared-failures (+ undeclared-failures (count (filter undeclared-failure? parts)))
+ terminal-error (terminal-error-message profile parts undeclared-failures)]
(cond
- (reduced? result')
+ @consumer-stopped?
;; consumer signalled early termination (e.g. client disconnect / cancellation)
- (assoc loop-state :status :reduced :finish-reason :reduced :result @result')
+ (assoc loop-state :status :reduced :finish-reason :reduced :result (unreduced result'))
terminal-error
(let [result'' (rf result' (terminal-error-text-part terminal-error))]
@@ -680,7 +718,11 @@
:result (rf result'' (final-state-part @memory-atom))))))
(should-continue? iteration max-iter terminal-tools parts)
- (assoc loop-state :result result' :iteration (inc iteration) :sent-parts input-parts)
+ (assoc loop-state
+ :result result'
+ :iteration (inc iteration)
+ :sent-parts input-parts
+ :undeclared-failures undeclared-failures)
:else
(let [reason (finish-reason iteration max-iter terminal-tools parts)]
@@ -804,12 +846,11 @@
:ai/state (:state opts)}
(let [agent (init-agent opts)
usage-atom (atom {})
- {result :result
- iteration :iteration
- finish-reason :finish-reason} (->> (initial-loop-state agent rf init usage-atom)
- (iterate loop-step)
- (drop-while #(= :continue (:status %)))
- first)]
+ {:keys [result iteration finish-reason consumer-stopped?]}
+ (->> (initial-loop-state agent rf init usage-atom)
+ (iterate loop-step)
+ (drop-while #(= :continue (:status %)))
+ first)]
(analytics/observe! :metabase-metabot/agent-iterations labels iteration)
;; Materialized turn-level outputs for evals: token usage, why the loop
;; stopped, and the final agent state (link registry / generated entities).
@@ -817,8 +858,9 @@
(ait/record! {:ai/finish-reason finish-reason
:ai/usage @usage-atom
:ai/final-state (some-> (:memory-atom agent) deref :state)}))
- ;; A :reduced stop (client disconnect / cancellation) has already terminated
- ;; the reducing fn — stepping it again would violate the transducer contract.
+ ;; A consumer that stopped (client disconnect / cancellation) has terminated the
+ ;; reducing fn, at whichever write it stopped, and stepping it again would violate
+ ;; the transducer contract.
;; Append trailing parts only on a normal finish: the debug log, the
;; loop's finish reason, and the eval-session pointer that lets the
;; harness locate the per-session `<session-id>.jsonl` (it reads that
@@ -827,8 +869,8 @@
;; in-process `capture-reducible`/`capturing` path is capture-active but
;; deliberately file-less (session id nil, read `:trace`), so emitting a
;; pointer there would name a `<nil>.jsonl` that never exists.
- (if (= :reduced finish-reason)
- result
+ (if @consumer-stopped?
+ (unreduced result)
(-> result
(cond->
(and debug? (seq @*debug-log*)) (rf (debug-log-part @*debug-log*))
diff --git a/src/metabase/metabot/persistence.clj b/src/metabase/metabot/persistence.clj
index e11fcf272f5..f1450faacf8 100644
--- a/src/metabase/metabot/persistence.clj
+++ b/src/metabase/metabot/persistence.clj
@@ -13,6 +13,7 @@
[metabase.metabot.self.core :as self.core]
[metabase.metabot.settings :as metabot.settings]
[metabase.metabot.tools :as metabot.tools]
+ [metabase.metabot.tools.runtime :as tools.runtime]
[metabase.metabot.used-tables :as used-tables]
[metabase.util :as u]
[metabase.util.json :as json]
@@ -102,8 +103,10 @@
;; failure — the table that was missing, the argument to repair — on
;; every replay. One field, two audiences; the model wins, because the
;; live SSE channel has its own and the whole point of the text is that
- ;; the agent can act on it.
- (assoc :errorText (str (get-in output [:result :output])))))
+ ;; the agent can act on it. A failure that no tool declared is stored as
+ ;; the neutral sentence `tools.runtime/replay-text` gives it, because its
+ ;; live text tells the model what to do in a turn that will have ended.
+ (assoc :errorText (str (tools.runtime/replay-text (:result output))))))
:data {:type (str "data-" (or (:data-type part) "data"))
:data (:data part)}
(do (log/warn "Dropping internal part with no v2 storage representation"
diff --git a/src/metabase/metabot/self/core.clj b/src/metabase/metabot/self/core.clj
index 4d84b2af858..3bef8db9975 100644
--- a/src/metabase/metabot/self/core.clj
+++ b/src/metabase/metabot/self/core.clj
@@ -1090,11 +1090,10 @@
(catch Throwable e
;; `invoke` does not throw. If something gets past it, pairing the
;; call with a result still matters more than the diagnosis, so
- ;; log and report the same shape a server fault reports.
+ ;; log it and report it the way `invoke` reports a failure.
(log/error e "Tool invocation escaped the runtime"
{:tool-name safe-name})
- {:output "This call failed and the user was shown the error (:internal). Don't retry it."
- :error {:class :unrecoverable :code :internal}}))))))
+ (tools.runtime/render (tools.error/classify e) (keys tools))))))))
results (collect-tool-result tool-call-id tool-name outcome)]
(when (ait/capture-active?)
(ait/record! {:ai/tool-output results}))
diff --git a/src/metabase/metabot/tools/construct.clj b/src/metabase/metabot/tools/construct.clj
index 8d6e9ad9672..d5e02a8d4b9 100644
--- a/src/metabase/metabot/tools/construct.clj
+++ b/src/metabase/metabot/tools/construct.clj
@@ -1353,7 +1353,7 @@
:chart-type
chart-type->keyword)
:table)
- ;; Declared pipeline errors and refusals are recoverable; anything else ends the turn.
+ ;; Declared pipeline errors and refusals are recoverable; anything else is `:internal`.
structured (:structured-output (tools/with-pipeline-errors
(execute-representations-query query {:preprocess-check? true})))
;; A query built only to be read needs run_query, so without it the chart is the one way
diff --git a/src/metabase/metabot/tools/core.clj b/src/metabase/metabot/tools/core.clj
index 9e02b6e1edc..9056d079f3f 100644
--- a/src/metabase/metabot/tools/core.clj
+++ b/src/metabase/metabot/tools/core.clj
@@ -15,12 +15,13 @@
to …\"}` convention: a failure is thrown, so forgetting to handle one cannot silently hand the
model a sentence nobody wrote.
- Everything a tool does about a failure is written at the call site, and the default — doing
- nothing — is that the failure ends the turn:
+ Everything a tool does about a failure is written at the call site. A failure it does nothing
+ about is undeclared: the model learns only that the call failed, and the agent loop decides
+ whether the turn goes on. What a tool can do instead:
- *relay a foreign error*: a converter ([[with-entity]], [[with-pipeline-errors]]) turns one known
shape into a declared recoverable error. A shape a converter does not recognise passes through
- unchanged and is therefore unrecoverable.
+ unchanged and is therefore undeclared.
- *do one thing per item*: implement [[BatchedTool]] as well, and let [[handle]] delegate to
[[handle-each]]. A declared recoverable error then becomes that item's contribution instead of
ending the turn.
@@ -120,7 +121,10 @@
[:args :any]
[:scope {:optional true} [:maybe :string]]
[:metabot/capabilities {:optional true} [:maybe [:set :keyword]]]
- [:metabot/title-fn {:optional true} [:maybe fn?]]])
+ [:metabot/title-fn {:optional true} [:maybe fn?]]
+ ;; Set by a tool that saves something. A failure it did not declare may have come after the save,
+ ;; so `metabase.metabot.tools.runtime/invoke` ends the turn instead of letting the model try again.
+ [:metabot/writes? {:optional true} [:maybe :boolean]]])
(mr/def ::result
"What [[handle]] returns. Success only — a failure is thrown.
diff --git a/src/metabase/metabot/tools/create_alert.clj b/src/metabase/metabot/tools/create_alert.clj
index 0d95bef52ca..390c3ca075e 100644
--- a/src/metabase/metabot/tools/create_alert.clj
+++ b/src/metabase/metabot/tools/create_alert.clj
@@ -134,10 +134,11 @@
(defrecord CreateAlertTool []
tools/Tool
(declaration [_]
- {:name "create_alert"
- :description tool-description
- :scope scope/agent-alert-create
- :args alert-schema})
+ {:name "create_alert"
+ :description tool-description
+ :scope scope/agent-alert-create
+ :args alert-schema
+ :metabot/writes? true})
(handle [_ {:keys [card_id send_condition schedule send_once]} ctx]
(let [channel (current-slack-channel ctx)
diff --git a/src/metabase/metabot/tools/create_dashboard_subscription.clj b/src/metabase/metabot/tools/create_dashboard_subscription.clj
index 7aa9fdc5765..6506b4a33aa 100644
--- a/src/metabase/metabot/tools/create_dashboard_subscription.clj
+++ b/src/metabase/metabot/tools/create_dashboard_subscription.clj
@@ -27,12 +27,13 @@
(defrecord SlackbotCreateDashboardSubscriptionTool []
tools/Tool
(declaration [_]
- {:name "create_dashboard_subscription"
- :description tool-description
- :scope scope/agent-dashboard-subscribe
- :args [:map {:closed true}
- [:dashboard_id :int]
- [:schedule tools.create-alert/schedule-schema]]})
+ {:name "create_dashboard_subscription"
+ :description tool-description
+ :scope scope/agent-dashboard-subscribe
+ :args [:map {:closed true}
+ [:dashboard_id :int]
+ [:schedule tools.create-alert/schedule-schema]]
+ :metabot/writes? true})
(handle [_ {:keys [dashboard_id schedule]} ctx]
(tools.subscriptions/create-dashboard-subscription!
diff --git a/src/metabase/metabot/tools/error.clj b/src/metabase/metabot/tools/error.clj
index ca2a571da12..71efeae710d 100644
--- a/src/metabase/metabot/tools/error.clj
+++ b/src/metabase/metabot/tools/error.clj
@@ -10,10 +10,11 @@
| `:unrecoverable` | everything else | the user | `:user-message`? |
Recoverable is opt-in and unrecoverable is the default, so the safe path is the one you get by
- doing nothing: an exception nobody declared recoverable ends the turn rather than handing the
- model text that nobody wrote for it. An unrecoverable error has no `:message` key at all, which
- makes forwarding a foreign exception message to the agent impossible rather than merely
- discouraged.
+ doing nothing: an exception nobody declared is unrecoverable, with a code from [[classify]], and
+ the model never reads text that nobody wrote for it. An unrecoverable error has no `:message` key
+ at all, which makes forwarding a foreign exception message to the agent impossible rather than
+ merely discouraged. The agent still reads a fixed sentence for an `:internal` failure, so that it
+ can try another way when the agent loop lets the turn go on.
Each recoverable error is declared once with [[defrecoverable]], which records it in the catalog
here. The catalog is what lets one test cover every declared error, and what lets a converter
@@ -123,11 +124,12 @@
cause)))
(defn unrecoverable!
- "Throw an unrecoverable ToolError with `code`, ending the turn.
+ "Throw an unrecoverable ToolError with `code`, ending the turn for any code a tool passes.
Always throws; never returns. Tool code passes a qualified keyword, so the log and the
model-facing template both say which namespace gave up; the runtime's own codes are an
- unqualified closed set (`metabase.metabot.tools.runtime/runtime-codes`).
+ unqualified closed set (`metabase.metabot.tools.runtime/runtime-codes`). Among those, the agent
+ loop can let a turn go on after `:internal`, the code of an ordinary exception no tool declared.
(unrecoverable! ::no-sql-permission)
(unrecoverable! ::no-sql-permission
@@ -322,8 +324,8 @@
A third audience, decided the way the other two are — by the class. The model's `:output` is not
it: that text is written to be acted on (\"`uris` must be an array of 1 to 5 URI strings\", \"the
- user was shown the error (:internal)\"), and someone reading their own conversation is not the one
- acting on it.
+ user was shown the error (:scope-denied)\"), and someone reading their own conversation is not the
+ one acting on it.
- `:unrecoverable` gets the `:user-message` the tool chose, because this is the failure the user
has to act on, and the generic sentence when the tool chose none.
@@ -380,9 +382,15 @@
"The ToolError for `e`.
An exception carrying [[error-key]] in its ex-data keeps the class its producer chose; everything
- else is unrecoverable `:internal`. Reads the top-level exception only and does not walk the cause
- chain, so wrapping a declared error in an undeclared one demotes it — which is the safe
- direction."
+ else is unrecoverable, with the code `:jvm-error` for a JVM error such as running out of memory or
+ stack, `:interrupted` for an interrupted thread, and `:internal` for an ordinary exception. The
+ model can't work around either of the first two. Reads the top-level exception only and does not
+ walk the cause chain, so wrapping a declared error in an undeclared one demotes it, which is the
+ safe direction."
[e]
(or (get (ex-data e) error-key)
- {:class :unrecoverable :code :internal}))
+ {:class :unrecoverable
+ :code (condp instance? e
+ VirtualMachineError :jvm-error
+ InterruptedException :interrupted
+ :internal)}))
diff --git a/src/metabase/metabot/tools/runtime.clj b/src/metabase/metabot/tools/runtime.clj
index 8e0eae2db11..b5a082e02f3 100644
--- a/src/metabase/metabot/tools/runtime.clj
+++ b/src/metabase/metabot/tools/runtime.clj
@@ -14,8 +14,10 @@
checks it.
Codes come in two flavours. Tool code uses qualified keywords, so a code says which namespace
- raised it. The runtime's own codes are a closed, unqualified vocabulary — [[runtime-codes]] — and
- the agent loop adds one more (`:repeated-error`)."
+ raised it. The runtime's own codes are a closed, unqualified vocabulary, [[runtime-codes]].
+ Every unrecoverable failure ends the turn except `:internal`, the code of an ordinary exception no
+ tool declared: the agent loop lets a turn survive one of those, or none when its profile requires
+ a tool call. A tool that writes gets none either: its undeclared failure is an [[unconfirmed-write]]."
(:require
[clojure.string :as str]
[malli.core :as mc]
@@ -43,11 +45,32 @@
"Every error code the runtime itself produces, with the class each one takes.
Written down as data so the agent loop, the adapters and the tests share one list rather than
- three sets of literal keywords. `:repeated-error` is the agent loop's and is not raised here."
+ three sets of literal keywords."
{:unknown-tool :validation
:invalid-arguments :validation
:scope-denied :unrecoverable
- :internal :unrecoverable})
+ :internal :unrecoverable
+ :jvm-error :unrecoverable
+ :interrupted :unrecoverable
+ :unconfirmed-write :unrecoverable})
+
+(defn undeclared?
+ "Whether `code` is `:internal`, the code of an ordinary exception that no tool declared."
+ [code]
+ (= :internal code))
+
+(defn- unconfirmed-write
+ "The ToolError to report for `error` from a tool whose declaration sets `:metabot/writes?`.
+
+ A failure that no tool declared, coded by `tools.error/classify`, may have come after the write. It
+ becomes `:unconfirmed-write`, which ends the turn and tells the user to check before trying again,
+ so the model can't write a second time. A declared error passes through unchanged."
+ [{:keys [code] :as error}]
+ (if (#{:internal :jvm-error :interrupted} code)
+ {:class :unrecoverable
+ :code :unconfirmed-write
+ :user-message (tru "Something went wrong. The change may have been saved, so check before you try again.")}
+ error))
;;; ------------------------------------------------ Schemas -------------------------------------------------------
@@ -152,17 +175,24 @@
;;; ------------------------------------------------ Rendering -----------------------------------------------------
+(def ^:private undeclared-failure-sentence
+ "How the model's text about an ordinary exception that no tool declared starts, and all that a later
+ turn replays of it (see [[replay-text]])."
+ "This call failed with an unexpected error.")
+
(defn- unrecoverable-output
- "What the model is told about an unrecoverable failure: the code, and not to retry.
-
- Nothing else. The user is being shown the real error on another channel, so the model's job here
- is to stop rather than to explain. The code is safe to include because it is ours — a closed
- vocabulary we wrote — and it gives the model something stable to refer to if it says anything at
- all. Note that the model only ever sees this *within the turn*, to pair the remaining tool calls
- of the iteration with results; a turn that ends in an error event is dropped from history whole,
- so later turns never read it."
+ "What the model is told about an unrecoverable failure, which never includes the failure's own text.
+
+ `:internal` is an ordinary exception that no tool declared. The agent loop can carry on after one,
+ so the model is told to try another way rather than make the same call again.
+
+ For any other code the user is shown the real error on another channel, so the model's job is to
+ stop rather than to explain. The code is safe to include because it is ours, a closed vocabulary
+ we wrote, and it gives the model something stable to refer to if it says anything at all."
[code]
- (str "This call failed and the user was shown the error (" code "). Don't retry it."))
+ (if (undeclared? code)
+ (str undeclared-failure-sentence " Don't retry the same call. Try another approach, or tell the user it failed.")
+ (str "This call failed and the user was shown the error (" code "). Don't retry it.")))
(def ^:private leak-patterns
"Shapes that mean a raw exception or EDN dump reached agent-bound text: a stack-trace frame,
@@ -189,7 +219,7 @@
| ---------------- | --------------------------------------------------------- |
| `:validation` | the message |
| `:recoverable` | the message, then each kept recovery step on its own line |
- | `:unrecoverable` | a fixed sentence carrying only the code |
+ | `:unrecoverable` | a fixed sentence, with the code unless it is `:internal` |
Failure text is capped at [[max-output-length]] characters.
@@ -218,6 +248,17 @@
:error (cond-> {:class class :code code}
user-message (assoc :user-message user-message))}))
+(defn replay-text
+ "The text that a later turn replays to the model for the failure [[::outcome]] `outcome`.
+
+ An ordinary exception that no tool declared replays as one neutral sentence. Its live text also
+ tells the model not to make the same call again, and that holds only for the turn that failed: by a
+ later turn the cause may be gone. Every other failure replays the text the model read live."
+ [{:keys [output error]}]
+ (if (undeclared? (:code error))
+ undeclared-failure-sentence
+ output))
+
;;; ------------------------------------------------ Invocation ----------------------------------------------------
(defn- validation-ex
@@ -331,7 +372,8 @@
3. the scope is checked, and a denial is unrecoverable;
4. the tool runs with the dynamic vars bound from `ctx`, through `tools.core/call`;
5. its result is validated against `tools.core/result` and its `:output` rendered to a string;
- 6. any exception is classified, logged in full, and rendered for its audience."
+ 6. any exception is classified, logged in full, and rendered for its audience; a tool that writes
+ has a failure no tool declared reported as an [[unconfirmed-write]]."
[entries ctx tool-name args]
(let [tool-names (:tool-names ctx)]
(try
@@ -343,6 +385,7 @@
(->> (checked-result tool-name))
rendered-output))
(catch Throwable e
- (let [error (tools.error/classify e)]
+ (let [error (cond-> (tools.error/classify e)
+ (get-in entries [tool-name :declaration :metabot/writes?]) unconfirmed-write)]
(log-failure! tool-name error e)
(render error tool-names))))))
diff --git a/src/metabase/metabot/tools/save_entity.clj b/src/metabase/metabot/tools/save_entity.clj
index 333a07224c7..3b9e15cde28 100644
--- a/src/metabase/metabot/tools/save_entity.clj
+++ b/src/metabase/metabot/tools/save_entity.clj
@@ -350,10 +350,11 @@
(defrecord SaveEntityTool []
tools/Tool
(declaration [_]
- {:name "save_entity"
- :description tool-description
- :scope scope/agent-question-create
- :args save-entity-schema})
+ {:name "save_entity"
+ :description tool-description
+ :scope scope/agent-question-create
+ :args save-entity-schema
+ :metabot/writes? true})
(handle [_ {:keys [chart_id destination description] question-name :name} _ctx]
(let [{:keys [dataset_query display]} (resolve-chart chart_id)
diff --git a/src/metabase/metabot/tools/subscriptions.clj b/src/metabase/metabot/tools/subscriptions.clj
index 88c11b81dbf..b2c1f86b840 100644
--- a/src/metabase/metabot/tools/subscriptions.clj
+++ b/src/metabase/metabot/tools/subscriptions.clj
@@ -105,13 +105,14 @@
(defrecord CreateDashboardSubscriptionTool []
tools/Tool
(declaration [_]
- {:name "create_dashboard_subscription"
- :description tool-description
- :scope scope/agent-dashboard-subscribe
- :args [:map {:closed true}
- [:dashboard_id :int]
- [:slack_channel [:string {:min 1}]]
- [:schedule tools.create-alert/schedule-schema]]})
+ {:name "create_dashboard_subscription"
+ :description tool-description
+ :scope scope/agent-dashboard-subscribe
+ :args [:map {:closed true}
+ [:dashboard_id :int]
+ [:slack_channel [:string {:min 1}]]
+ [:schedule tools.create-alert/schedule-schema]]
+ :metabot/writes? true})
(handle [_ {:keys [dashboard_id slack_channel schedule]} _ctx]
(tools.create-alert/check-slack-configured!)
diff --git a/test/metabase/metabot/agent/core_test.clj b/test/metabase/metabot/agent/core_test.clj
index dbe153a39f7..6abf45c9aa6 100644
--- a/test/metabase/metabot/agent/core_test.clj
+++ b/test/metabase/metabot/agent/core_test.clj
@@ -1,6 +1,7 @@
(ns metabase.metabot.agent.core-test
(:require
[clj-http.client :as http]
+ [clojure.core.async :as a]
[clojure.string :as str]
[clojure.test :refer :all]
[metabase.ai-tracing.core :as ait]
@@ -14,6 +15,7 @@
[metabase.metabot.agent.core :as agent]
[metabase.metabot.agent.memory :as memory]
[metabase.metabot.agent.profiles :as profiles]
+ [metabase.metabot.api :as metabot.api]
[metabase.metabot.config :as metabot.config]
[metabase.metabot.persistence :as metabot.persistence]
[metabase.metabot.scope :as scope]
@@ -24,12 +26,15 @@
[metabase.metabot.self.openrouter :as openrouter]
[metabase.metabot.settings :as metabot.settings]
[metabase.metabot.test-util :as mut]
+ [metabase.metabot.tools.core :as tools.core]
[metabase.metabot.tools.legacy :as tools.legacy]
[metabase.metabot.tools.search :as metabot-search]
+ [metabase.metabot.tools.sql.create :as create-sql-query-tools]
[metabase.metabot.util :as metabot.u]
[metabase.permissions.core :as perms]
[metabase.permissions.models.permissions-group :as perms-group]
[metabase.test :as mt]
+ [metabase.util :as u]
[metabase.util.json :as json]
[toucan2.core :as t2]))
@@ -293,25 +298,40 @@
user-message (assoc-in [:error :user-message] user-message))})
(deftest terminal-error-message-test
- (let [denial [{:type :tool-input :id "a" :function "create_sql_query"}
- (unrecoverable-part "a" "No native permission.")]]
+ (let [denial [{:type :tool-input :id "a" :function "create_sql_query"}
+ (unrecoverable-part "a" "No native permission.")]
+ undeclared {:type :tool-output :id "d" :result mut/internal-failure}]
(testing "reads the user-facing message off an unrecoverable failure"
- (is (= "No native permission." (#'agent/terminal-error-message denial))))
+ (is (= "No native permission." (#'agent/terminal-error-message {} denial 0))))
(testing "a recoverable failure is not terminal — the model can still self-correct"
(is (nil? (#'agent/terminal-error-message
+ {}
[{:type :tool-output :id "b" :result {:output "syntax error"
- :error {:class :recoverable :code :x}}}]))))
+ :error {:class :recoverable :code :x}}}]
+ 0))))
(testing "and neither is a success"
(is (nil? (#'agent/terminal-error-message
- [{:type :tool-output :id "b" :result {:output "fine"}}]))))
+ {}
+ [{:type :tool-output :id "b" :result {:output "fine"}}]
+ 0))))
(testing "an unrecoverable failure with no :user-message gets the generic one, so the user is
always told something"
(is (= "Something went wrong."
- (#'agent/terminal-error-message [(unrecoverable-part "c" nil)]))))
+ (#'agent/terminal-error-message {} [(unrecoverable-part "c" nil)] 0))))
(testing "the first denial wins when an iteration produces several"
(is (= "first" (#'agent/terminal-error-message
+ {}
[(unrecoverable-part "a" "first")
- (unrecoverable-part "b" "second")]))))
+ (unrecoverable-part "b" "second")]
+ 0))))
+ (testing "a failure no tool declared is terminal only once it is the turn's second"
+ (is (nil? (#'agent/terminal-error-message {} [undeclared] 1)))
+ (is (= "Something went wrong." (#'agent/terminal-error-message {} [undeclared] 2)))
+ (testing "or its first, in a profile that requires a tool call"
+ (is (= "Something went wrong."
+ (#'agent/terminal-error-message {:required-tool-call? true} [undeclared] 1))))
+ (testing "and a denial in the same iteration still shows its own message"
+ (is (= "No native permission." (#'agent/terminal-error-message {} (into [undeclared] denial) 2)))))
(testing "should-continue? is unaffected — the gate lives in loop-step, per profile"
(is (#'agent/should-continue? 0 20 #{} denial)))))
@@ -324,37 +344,57 @@
[{:type "code_editor"
:buffers [{:id "buf-1" :source {:language "sql" :database_id nil} :cursor {:line 0 :column 0}}]}])
-(defn- run-sql-denial-turn!
- "Run one turn whose first LLM response calls `create_sql_query` against `database-id` while the user views
- `viewing`, by default a SQL editor."
- [profile-id database-id & {:keys [viewing] :or {viewing sql-editor-viewing}}]
- (let [call-count (atom 0)]
+(defn- run-turn!
+ "Run one turn of `profile-id` that starts with the user's `message`. The LLM returns `responses` in order and then
+ answers \"Sorry.\" in text. `context` is the request's context. `writer`, when given, receives the turn as SSE lines,
+ the way the chat endpoint streams it. Return the LLM requests and the turn's parts."
+ [profile-id message responses & {:keys [context writer]}]
+ (let [requests (atom [])]
(mt/with-temporary-setting-values [llm-providers llm.tu/default-connections
llm-metabot-provider test-provider]
;; the tool executor lives inside `call-llm`, so redef the transport to let the tool run
- (mt/with-dynamic-fn-redefs [openrouter/openrouter (fn [_]
- (if (= 1 (swap! call-count inc))
- (mut/mock-llm-response
- [{:type :tool-input
- :id "t1"
- :function "create_sql_query"
- :arguments {:database_id database-id
- :sql_query "SELECT 1"
- :title "Results"}}])
- (mut/mock-llm-response [{:type :text :text "Sorry."}])))]
- (let [parts (into [] (run-agent-loop
- {:messages [{:role :user :content "Query that database"}]
- :state {}
- :profile-id profile-id
- :context {:capabilities ["permission:write_sql_queries"]
- :user_is_viewing viewing}}))]
- {:llm-calls @call-count :parts parts})))))
+ (mt/with-dynamic-fn-redefs [openrouter/openrouter (fn [request]
+ (mut/mock-llm-response
+ (get responses (dec (count (swap! requests conj request)))
+ [{:type :text :text "Sorry."}])))]
+ (let [turn (run-agent-loop
+ {:messages [{:role :user :content message}]
+ :state {}
+ :profile-id profile-id
+ :context context})
+ parts (if writer
+ (let [streamed (atom [])]
+ (transduce (comp (u/tee-xf streamed) (self.core/parts->aisdk-sse-xf)) writer turn)
+ @streamed)
+ (into [] turn))]
+ {:llm-calls (count @requests) :requests @requests :parts parts})))))
+
+(defn- run-sql-turn!
+ "Run one turn whose LLM responses call `create_sql_query` against `database-id` and then answer in text, while the
+ user views `viewing`, by default a SQL editor. `calls` holds the call IDs of each response, by default one response
+ with one call. `writer` is as for [[run-turn!]]."
+ [profile-id database-id & {:keys [viewing calls writer] :or {viewing sql-editor-viewing, calls [["t1"]]}}]
+ (run-turn! profile-id
+ "Query that database"
+ (mapv (fn [ids]
+ (mapv (fn [id]
+ {:type :tool-input
+ :id id
+ :function "create_sql_query"
+ :arguments {:database_id database-id
+ :sql_query "SELECT 1"
+ :title "Results"}})
+ ids))
+ calls)
+ :context {:capabilities ["permission:write_sql_queries"]
+ :user_is_viewing viewing}
+ :writer writer))
(deftest missing-sql-editor-ends-a-forced-tool-call-turn-test
(testing "the SQL profile ends the turn when no SQL editor is open, and tells the user why"
(mt/with-temp [:model/Database {db-id :id} {:engine :h2}]
(mt/with-current-user (mt/user->id :crowberto)
- (let [{:keys [llm-calls parts]} (run-sql-denial-turn! :sql db-id :viewing [])]
+ (let [{:keys [llm-calls parts]} (run-sql-turn! :sql db-id :viewing [])]
(is (= 1 llm-calls))
(is (= :terminal-error (:finish-reason (last parts))))
(is (some #(and (= :text (:type %))
@@ -374,7 +414,7 @@
(perms/set-database-permission! (perms-group/all-users) unreadable-db :perms/create-queries :no)
(mt/with-current-user (mt/user->id :rasta)
(testing ":sql has no notebook builder to fall back to, so the loop stops on the denial"
- (let [{:keys [llm-calls parts]} (run-sql-denial-turn! :sql builder-db)]
+ (let [{:keys [llm-calls parts]} (run-sql-turn! :sql builder-db)]
(is (= 1 llm-calls)
"one call, not the profile's 20 iterations")
(is (= :terminal-error (:finish-reason (last parts))))
@@ -386,7 +426,7 @@
"state data part still closes the turn")))
(testing (str ":internal has the notebook builder, so the agent is told to build the question "
"there and the turn continues")
- (let [{:keys [llm-calls parts]} (run-sql-denial-turn! :internal builder-db)]
+ (let [{:keys [llm-calls parts]} (run-sql-turn! :internal builder-db)]
(is (= 2 llm-calls))
(is (=? {:type :tool-output
:result {:error {:class :recoverable}
@@ -395,11 +435,11 @@
(is (not= :terminal-error (:finish-reason (last parts))))))
(testing ":internal-sql is :internal without the notebook builder, so the turn stops on the denial"
(mt/with-temp-env-var-value! [mb-metabot-eval-profiles-enabled true]
- (let [{:keys [llm-calls parts]} (run-sql-denial-turn! :internal-sql builder-db)]
+ (let [{:keys [llm-calls parts]} (run-sql-turn! :internal-sql builder-db)]
(is (= 1 llm-calls))
(is (= :terminal-error (:finish-reason (last parts)))))))
(testing "a database the user cannot read at all stops the turn the same way"
- (let [{:keys [llm-calls parts]} (run-sql-denial-turn! :sql unreadable-db)]
+ (let [{:keys [llm-calls parts]} (run-sql-turn! :sql unreadable-db)]
(is (= 1 llm-calls)
"the read-check denial is terminal too -- otherwise the stricter permission loops")
(is (= :terminal-error (:finish-reason (last parts))))
@@ -407,9 +447,88 @@
(str/includes? (:text %) "do not have access to this database"))
parts))))
(testing "a database the user can query natively is not denied"
- (let [{:keys [parts]} (run-sql-denial-turn! :sql native-db)]
+ (let [{:keys [parts]} (run-sql-turn! :sql native-db)]
(is (= :terminal-tool (:finish-reason (last parts))))))))))
+(deftest a-turn-survives-one-undeclared-failure-test
+ (mt/with-current-user (mt/user->id :crowberto)
+ (mt/with-dynamic-fn-redefs [create-sql-query-tools/create-sql-query
+ (fn [_] (throw (ex-info "Connection reset by peer" {})))]
+ (testing "the model reads only that the call failed, and answers"
+ (let [{:keys [llm-calls requests parts]} (run-sql-turn! :internal (mt/id))]
+ (is (= 2 llm-calls))
+ (is (some #{(:output mut/internal-failure)} (tree-seq coll? seq (:input (second requests)))))
+ (is (= :stop (:finish-reason (last parts))))))
+ (testing "the turn ends with the generic message"
+ (doseq [[label profile-id calls] [["on the second, in a later iteration" :internal [["t1"] ["t2"]]]
+ ["on the second, in the same iteration" :internal [["t1" "t2"]]]
+ ["on the first, in a profile that requires a tool call" :sql [["t1"]]]]]
+ (testing label
+ (let [{:keys [llm-calls parts]} (run-sql-turn! profile-id (mt/id) :calls calls)]
+ (is (= (count calls) llm-calls))
+ (is (= :terminal-error (:finish-reason (last parts))))
+ (is (some #(= "Something went wrong." (:text %)) parts)))))))))
+
+(deftest a-jvm-error-or-an-interrupt-ends-the-turn-at-once-test
+ (mt/with-current-user (mt/user->id :crowberto)
+ (doseq [throwable [(StackOverflowError.) (InterruptedException.)]]
+ (testing (.getSimpleName (class throwable))
+ (mt/with-dynamic-fn-redefs [create-sql-query-tools/create-sql-query (fn [_] (throw throwable))]
+ (let [{:keys [llm-calls parts]} (run-sql-turn! :internal (mt/id))]
+ (is (= 1 llm-calls))
+ (is (= :terminal-error (:finish-reason (last parts))))
+ (is (some #(= "Something went wrong." (:text %)) parts))))))))
+
+(defn- note-tool
+ "A tool named `save_note` that fails on every call with an exception no tool declared. Its declaration says it
+ writes when `writes?` is true."
+ [writes?]
+ (reify tools.core/Tool
+ (declaration [_]
+ (cond-> {:name "save_note" :description "Saves a note." :args [:map {:closed true}]}
+ writes? (assoc :metabot/writes? true)))
+ (handle [_ _args _ctx]
+ (throw (ex-info "Connection reset after the commit" {})))))
+
+(deftest a-tool-that-writes-gets-no-second-try-test
+ (mt/with-current-user (mt/user->id :crowberto)
+ (let [get-profile (mt/original-fn #'profiles/get-profile)]
+ (doseq [[label writes? expected-calls finish-reason text]
+ [["a tool that writes ends the turn at once, since the write may have happened" true 1 :terminal-error
+ "Something went wrong. The change may have been saved, so check before you try again."]
+ ["a tool that only reads keeps its second try" false 2 :stop "Sorry."]]]
+ (testing label
+ (mt/with-dynamic-fn-redefs [profiles/get-profile (fn [profile-id]
+ (assoc (get-profile profile-id)
+ :tools [(note-tool writes?)]))]
+ (let [{:keys [llm-calls parts]}
+ (run-turn! :internal "Save a note" [[{:type :tool-input :id "t1" :function "save_note"
+ :arguments {}}]])]
+ (is (= expected-calls llm-calls))
+ (is (= finish-reason (:finish-reason (last parts))))
+ (is (= text (apply str (keep #(when (= :text (:type %)) (:text %)) parts)))
+ "what the user reads"))))))))
+
+(deftest the-turn-stops-when-the-client-leaves-during-a-tool-call-test
+ (mt/with-current-user (mt/user->id :crowberto)
+ (let [create-sql-query (mt/original-fn #'create-sql-query-tools/create-sql-query)]
+ (doseq [[outcome call] {"succeeds" create-sql-query
+ "fails with an undeclared exception" (fn [_] (throw (ex-info "Connection reset" {})))}]
+ (testing (str "the client leaves while a tool call runs, and the call " outcome)
+ (let [canceled-chan (a/promise-chan)
+ canceled? (volatile! false)]
+ (mt/with-dynamic-fn-redefs [create-sql-query-tools/create-sql-query (fn [args]
+ (a/>!! canceled-chan :canceled)
+ (call args))]
+ (let [{:keys [llm-calls parts]}
+ (run-sql-turn! :internal (mt/id)
+ :writer (#'metabot.api/streaming-writer-rf
+ (java.io.ByteArrayOutputStream.) canceled-chan canceled?))]
+ (is @canceled?)
+ (is (= 1 llm-calls))
+ (is (not-any? #(= :finish (:type %)) parts)
+ "the turn stops where the client left, without finishing")))))))))
+
(deftest blank-replacement-after-edit-test
;; A real user, not `mt/as-admin`: the SQL tools refuse a caller without a user id.
(mt/with-current-user (mt/user->id :crowberto)
diff --git a/test/metabase/metabot/persistence_test.clj b/test/metabase/metabot/persistence_test.clj
index d6aa2c515ff..63b458083fe 100644
--- a/test/metabase/metabot/persistence_test.clj
+++ b/test/metabase/metabot/persistence_test.clj
@@ -1484,7 +1484,15 @@
{:type :tool-output :id "c1"
:result {:output "Table 7 was not found. Call `search` for a table id."
:error {:class :recoverable
- :code :metabase.metabot.tools.recoverable.common/not-found}}}]))))))))
+ :code :metabase.metabot.tools.recoverable.common/not-found}}}]))))
+ (testing (str "a call that failed with an exception no tool declared replays one neutral sentence, "
+ "because the live text's instruction not to retry was for the turn that failed")
+ (is (= "This call failed with an unexpected error."
+ (tool-content [{:type :tool-input :id "c1" :function "search" :arguments {}}
+ {:type :tool-output :id "c1"
+ :result {:output (str "This call failed with an unexpected error. Don't retry the "
+ "same call. Try another approach, or tell the user it failed.")
+ :error {:class :unrecoverable :code :internal}}}]))))))))
(deftest history-hides-query-results-while-query-execution-is-off-test
(testing "a stored run_query result is replayed only while query execution is on, and nothing stored changes"
diff --git a/test/metabase/metabot/self_test.clj b/test/metabase/metabot/self_test.clj
index ad03cdbe7e9..a03c1d91081 100644
--- a/test/metabase/metabot/self_test.clj
+++ b/test/metabase/metabot/self_test.clj
@@ -672,6 +672,18 @@
:error {:class :unrecoverable :code :internal}}}
(last result))))))
+(deftest ^:parallel escaped-jvm-errors-keep-their-code-test
+ (testing "a JVM error or an interrupt that escapes a tool is reported with its own code"
+ (doseq [[throwable code] [[(StackOverflowError.) :jvm-error]
+ [(InterruptedException.) :interrupted]]]
+ (let [tools {"fail" {:tool-name "fail"
+ :doc "Throws."
+ :schema [:=> [:cat [:map {:closed true}]] :any]
+ :fn (fn [_] (throw throwable))}}
+ chunks (test-util/parts->aisdk-chunks [{:type :tool-input :id "call-1" :function "fail" :arguments {}}])]
+ (is (=? {:result {:error {:class :unrecoverable :code code}}}
+ (last (into [] (self.core/tool-executor-xf tools) chunks))))))))
+
(def ^:private failing-tool-exception
"What `failing-tool` throws. An atom so the tool can be a real var, which is what `adapt` accepts."
(atom nil))
@@ -2968,18 +2980,23 @@
"so does the text written for the reader"))))
(testing (str "persistence stores it as a failed step, keeping the MODEL's text: "
"`tool-part->llm-messages` replays `:errorText` to the model, so a generic "
- "sentence here would lose the authored failure on every replay")
- (let [stored (#'persistence/internal-parts->storable
- [{:type :tool-input :id "c1" :function "failer" :arguments {}}
- part])
- entry (first (filter #(= "tool-failer" (:type %)) stored))]
+ "sentence here would lose the authored failure on every replay. Only an exception "
+ "nothing classified is stored as one neutral sentence, because its live text tells "
+ "the model not to retry in a turn that will have ended")
+ (let [stored (#'persistence/internal-parts->storable
+ [{:type :tool-input :id "c1" :function "failer" :arguments {}}
+ part])
+ entry (first (filter #(= "tool-failer" (:type %)) stored))
+ replayed (if (= "internal" expected-code)
+ "This call failed with an unexpected error."
+ (:output outcome))]
(is (= "output-error" (:state entry)) (pr-str entry))
- (is (= (:output outcome) (:errorText entry)))
+ (is (= replayed (:errorText entry)))
(is (nil? (:output entry))
"a failed step stores no :output; its text is the :errorText")
(testing "and that text replays to the model verbatim"
;; a pair: the assistant's tool_call, then the `:tool` message the model reads
- (is (= (:output outcome)
+ (is (= replayed
(:content (last (persistence/tool-part->llm-messages entry)))))))))))))
(deftest a-rejected-argument-is-failed-at-every-consumer-test
diff --git a/test/metabase/metabot/test_util.clj b/test/metabase/metabot/test_util.clj
index 44b41a83165..7f1d66a2116 100644
--- a/test/metabase/metabot/test_util.clj
+++ b/test/metabase/metabot/test_util.clj
@@ -243,8 +243,9 @@
(t2/query ["SELECT no_such_column FROM metabase_field"])))
(def internal-failure
- "What the model reads when a call fails inside Metabase. The turn ends there, and the user gets the generic error."
- {:output "This call failed and the user was shown the error (:internal). Don't retry it."
+ "What the model reads when a call fails inside Metabase in a way no tool declared."
+ {:output (str "This call failed with an unexpected error. Don't retry the same call. "
+ "Try another approach, or tell the user it failed.")
:error {:class :unrecoverable :code :internal}})
;;; ──────────────────────────────────────────────────────────────────
diff --git a/test/metabase/metabot/tools/analyze_chart_test.clj b/test/metabase/metabot/tools/analyze_chart_test.clj
index 6c3a61d16f1..3ee9586e892 100644
--- a/test/metabase/metabot/tools/analyze_chart_test.clj
+++ b/test/metabase/metabot/tools/analyze_chart_test.clj
@@ -209,7 +209,7 @@
(str/join (repeat 150 "x")) (str "\"" (str/join (repeat 100 "x")) "...\""))))
(deftest ^:parallel analyze-chart-tool-unexpected-error-test
- (testing "an unexpected error ends the turn without reaching the model"
+ (testing "an unexpected error is `:internal`, so its text never reaches the model"
(mt/with-dynamic-fn-redefs [interestingness/compute-chart-stats (fn [& _] (throw (ex-info "boom" {})))]
(is (= {:class :unrecoverable :code :internal}
(failure "c1" (numeric-series-config)))))))
diff --git a/test/metabase/metabot/tools/autogen_dashboard_test.clj b/test/metabase/metabot/tools/autogen_dashboard_test.clj
index 85c6bd98290..404a4b44388 100644
--- a/test/metabase/metabot/tools/autogen_dashboard_test.clj
+++ b/test/metabase/metabot/tools/autogen_dashboard_test.clj
@@ -309,7 +309,7 @@
(invoke {:query_id "q-sql"} queries))))))))
(deftest unexpected-error-test
- (testing "an error that no one declared ends the turn"
+ (testing "an error that no one declared is `:internal`"
(mt/with-dynamic-fn-redefs [metabot.db/table (fn [_] (throw (ex-info "boom" {})))]
(is (= {:class :unrecoverable :code :internal}
(:error (invoke {:table_id 1})))))))
diff --git a/test/metabase/metabot/tools/charts_test.clj b/test/metabase/metabot/tools/charts_test.clj
index 790d02836bb..8e0da06601c 100644
--- a/test/metabase/metabot/tools/charts_test.clj
+++ b/test/metabase/metabot/tools/charts_test.clj
@@ -379,7 +379,7 @@
{}))))))))
(deftest chart-tools-undeclared-failure-test
- (testing "a failure that no error declares ends the turn instead of reaching the model"
+ (testing "a failure that no error declares is `:internal`, so its text never reaches the model"
(mt/with-dynamic-fn-redefs [create-chart-tools/create-chart (fn [_] (throw (ex-info "boom" {})))
edit-chart-tools/edit-chart (fn [_] (throw (ex-info "boom" {})))]
(is (=? {:error {:class :unrecoverable :code :internal}}
diff --git a/test/metabase/metabot/tools/construct_representations_test.clj b/test/metabase/metabot/tools/construct_representations_test.clj
index 34460cc7879..1eb6a34b98b 100644
--- a/test/metabase/metabot/tools/construct_representations_test.clj
+++ b/test/metabase/metabot/tools/construct_representations_test.clj
@@ -260,7 +260,7 @@
:source-table ["Sample" "PUBLIC" "ORDERS"]
:fields [["field" {} ["Sample" "PUBLIC" "ORDERS" "ID"]]]}]})
-(deftest export-validation-failures-end-metabot-turn-test
+(deftest export-validation-failures-are-internal-errors-test
(with-mp-and-stubs!
(fn []
(let [export-query (mt/original-fn #'repr.resolve/export-query)]
@@ -281,11 +281,7 @@
:title "Order IDs" :description "IDs of orders."})]
(is (nil? (:error (ex-data caught))))
(is (= :unrecoverable (:class (tools.error/classify caught))))
- (is (= {:output "This call failed and the user was shown the error (:internal). Don't retry it."
- :error {:class :unrecoverable :code :internal}}
- outcome))
- (is (= "Something went wrong."
- (#'agent/terminal-error-message [{:type :tool-output :result outcome}])))))))))
+ (is (= test-util/internal-failure outcome))))))))
(deftest metadata-lookup-failures-propagate-test
(with-mp-and-stubs!
@@ -363,9 +359,9 @@
:error {:class :recoverable
:code :metabase.metabot.tools.recoverable.pipeline/invalid-representations-query}}
outcome))
- (is (nil? (#'agent/terminal-error-message [{:type :tool-output :result outcome}])))))))))
+ (is (nil? (#'agent/terminal-error-message {} [{:type :tool-output :result outcome}] 0)))))))))
-(deftest metadata-lookup-failures-end-metabot-turn-test
+(deftest metadata-lookup-failures-are-internal-errors-test
(with-mp-and-stubs!
(fn []
(doseq [[label failure] (metadata-lookup-failures)]
@@ -379,11 +375,7 @@
"construct_notebook_query"
{:query orders-id-query
:title "Order IDs" :description "IDs of orders."})]
- (is (= {:output "This call failed and the user was shown the error (:internal). Don't retry it."
- :error {:class :unrecoverable :code :internal}}
- outcome))
- (is (= "Something went wrong."
- (#'agent/terminal-error-message [{:type :tool-output :result outcome}]))))))))))
+ (is (= test-util/internal-failure outcome)))))))))
(deftest implicit-join-happy-path-test
(testing "cross-table breakout gets auto-wired with a :source-field after repair"
diff --git a/test/metabase/metabot/tools/core_test.clj b/test/metabase/metabot/tools/core_test.clj
index 459873128a7..cd69ad012d9 100644
--- a/test/metabase/metabot/tools/core_test.clj
+++ b/test/metabase/metabot/tools/core_test.clj
@@ -237,7 +237,7 @@
(is (some? @inner)))))
(deftest ^:parallel only-declared-recoverables-are-captured-test
- (testing "an undeclared exception ends the turn and discards the items that loaded"
+ (testing "an undeclared exception fails the whole call and discards the items that loaded"
(is (thrown-with-msg?
clojure.lang.ExceptionInfo #"a real bug"
(tools/call (reify
diff --git a/test/metabase/metabot/tools/create_alert_test.clj b/test/metabase/metabot/tools/create_alert_test.clj
index 12b28ff50b5..44730b9b777 100644
--- a/test/metabase/metabot/tools/create_alert_test.clj
+++ b/test/metabase/metabot/tools/create_alert_test.clj
@@ -105,7 +105,10 @@
:code ::tools.create-alert/no-alert-permission
:user-message "You don't have permission to create alerts."}
(:error (create-alert (daily-alert card-id)))))))
- (testing "any other failure stays an internal error"
+ (testing "any other failure may have come after the alert was saved, so the user is told to check"
(mt/with-dynamic-fn-redefs [notification.api/create-notification! (fn [_] (throw (ex-info "boom" {})))]
- (is (= {:class :unrecoverable :code :internal}
+ (is (= {:class :unrecoverable
+ :code :unconfirmed-write
+ :user-message (str "Something went wrong. The change may have been saved, "
+ "so check before you try again.")}
(:error (create-alert (daily-alert card-id)))))))))))
diff --git a/test/metabase/metabot/tools/error_test.clj b/test/metabase/metabot/tools/error_test.clj
index 3498c680fa3..8ee678fb6fa 100644
--- a/test/metabase/metabot/tools/error_test.clj
+++ b/test/metabase/metabot/tools/error_test.clj
@@ -113,6 +113,11 @@
(tools.error/classify (ex-info "a plain failure" {:status-code 500}))))
(is (= {:class :unrecoverable :code :internal}
(tools.error/classify (RuntimeException. "boom")))))
+ (testing "a JVM error and an interrupt get codes of their own"
+ (is (= {:class :unrecoverable :code :jvm-error}
+ (tools.error/classify (StackOverflowError.))))
+ (is (= {:class :unrecoverable :code :interrupted}
+ (tools.error/classify (InterruptedException.)))))
(testing "classify reads the top-level exception only — wrapping a declared error demotes it,
which is the safe direction"
(let [declared (is (thrown? clojure.lang.ExceptionInfo (test-custom-status! {:id 1})))]
diff --git a/test/metabase/metabot/tools/explorations_test.clj b/test/metabase/metabot/tools/explorations_test.clj
index 01c7b84929f..0ce17f1e05e 100644
--- a/test/metabase/metabot/tools/explorations_test.clj
+++ b/test/metabase/metabot/tools/explorations_test.clj
@@ -189,7 +189,7 @@
:metric-name "Churn"
:dimension-ids ["plan-1"]
:candidates []}]}))))
- (testing "anything else that fails in validation still ends the turn"
+ (testing "anything else that fails in validation is `:internal`"
(is (= {:class :unrecoverable :code :internal}
(:error (add-groups-rejected-with {:something "else"}))))))
diff --git a/test/metabase/metabot/tools/recoverable/pipeline_test.clj b/test/metabase/metabot/tools/recoverable/pipeline_test.clj
index 0c6a267da8a..a8e60a1554e 100644
--- a/test/metabase/metabot/tools/recoverable/pipeline_test.clj
+++ b/test/metabase/metabot/tools/recoverable/pipeline_test.clj
@@ -2,8 +2,9 @@
"Guards the one gap `with-pipeline-errors` cannot close by itself.
The converter maps a pipeline `:error` code onto a declaration by name. A code with no declaration
- is not converted, so it becomes unrecoverable and ends the turn — silently, and only in production.
- This test reads the pipeline's own sources and fails while the gap is still cheap to close."
+ is not converted, so it becomes `:internal` and the model never reads the pipeline's message. That
+ happens silently, and only in production. This test reads the pipeline's own sources and fails
+ while the gap is still cheap to close."
(:require
[clojure.java.io :as io]
[clojure.set :as set]
@@ -70,7 +71,7 @@
(is (empty? missing)
(str "These representations-pipeline :error codes have no declaration in "
"metabase.metabot.tools.recoverable.pipeline, so `tools.core/with-pipeline-errors` "
- "will not convert them and they will end the turn instead of teaching the agent: "
+ "will not convert them, and the agent will read a fixed sentence instead of what went wrong: "
(pr-str (sort missing))
". Add a `defpipeline-error` for each, or decide it really is unrecoverable and say so "
"in a comment there."))))
diff --git a/test/metabase/metabot/tools/run_query_test.clj b/test/metabase/metabot/tools/run_query_test.clj
index d3c3581e58d..abbde0ffbb7 100644
--- a/test/metabase/metabot/tools/run_query_test.clj
+++ b/test/metabase/metabot/tools/run_query_test.clj
@@ -1215,12 +1215,12 @@
rejected-sql (mt/native-query {:query "SELECT * FROM NO_SUCH_TABLE"})]
(doseq [profile run-query-profiles]
(testing profile
- (testing "a run that fails inside Metabase ends the turn, and the user gets the generic error"
+ (testing "a run that fails inside Metabase tells the model only that it failed, and the model answers"
(binding [qp.pipeline/*execute* mut/reject-app-db-lookup]
- (is (= {:llm-calls 1
+ (is (= {:llm-calls 2
:result mut/internal-failure
- :text "Something went wrong."
- :finish :terminal-error}
+ :text "Done."
+ :finish :stop}
(run-turn! profile sql)))))
(testing "a query the database rejects reaches the model quoted, and the model answers"
(is (=? {:llm-calls 2
diff --git a/test/metabase/metabot/tools/runtime_test.clj b/test/metabase/metabot/tools/runtime_test.clj
index a229a47a52a..88c9ba12369 100644
--- a/test/metabase/metabot/tools/runtime_test.clj
+++ b/test/metabase/metabot/tools/runtime_test.clj
@@ -83,6 +83,23 @@
(fn [_ _] (throw (ex-info "SELECT * FROM secrets failed: clojure.lang.ExceptionInfo at line 1"
{:password "hunter2"})))))
+(def ^:private jvm-failing-tool
+ (tool {:name "jvm_failure" :description "Overflows its stack or gets interrupted."
+ :args [:map {:closed true} [:kind [:enum "overflow" "interrupt"]]]}
+ (fn [{:keys [kind]} _]
+ (throw (case kind
+ "overflow" (StackOverflowError.)
+ "interrupt" (InterruptedException.))))))
+
+(def ^:private writing-tool
+ (tool {:name "write" :description "Writes, then fails."
+ :args [:map {:closed true} [:declared {:optional true} :boolean]]
+ :metabot/writes? true}
+ (fn [{:keys [declared]} _]
+ (if declared
+ (tools.error/unrecoverable! ::out-of-ink {:user-message "Out of ink."})
+ (throw (ex-info "Connection reset after the commit" {}))))))
+
(def ^:private bad-shape-tool
(tool {:name "bad_shape" :description "Returns something that is not a result."
:args [:map {:closed true}]}
@@ -119,7 +136,8 @@
(def ^:private entries
(tools/entries [happy-tool recoverable-tool unrecoverable-tool silent-unrecoverable-tool
- crashing-tool bad-shape-tool scoped-tool memory-tool multi-item-tool]))
+ crashing-tool jvm-failing-tool writing-tool bad-shape-tool scoped-tool memory-tool
+ multi-item-tool]))
(def ^:private all-tool-names
(into #{"search" "read_resource"} (keys entries)))
@@ -221,13 +239,35 @@
(deftest ^:parallel an-undeclared-exception-is-unrecoverable-and-leaks-nothing-test
(let [outcome (invoke "crash" {})]
(is (= {:class :unrecoverable :code :internal} (:error outcome)))
- (is (= "This call failed and the user was shown the error (:internal). Don't retry it."
+ (is (= (str "This call failed with an unexpected error. Don't retry the same call. "
+ "Try another approach, or tell the user it failed.")
(:output outcome)))
(testing "neither the exception message nor its ex-data reaches the model"
(doseq [secret ["SELECT" "secrets" "clojure.lang" "hunter2"]]
(is (not (str/includes? (:output outcome) secret))
(str "leaked " secret))))))
+(deftest ^:parallel a-tool-that-writes-ends-the-turn-on-an-undeclared-failure-test
+ (testing "the write may have happened, so the user is told to check before trying again"
+ (is (= {:output "This call failed and the user was shown the error (:unconfirmed-write). Don't retry it."
+ :error {:class :unrecoverable
+ :code :unconfirmed-write
+ :user-message (str "Something went wrong. The change may have been saved, "
+ "so check before you try again.")}}
+ (invoke "write" {}))))
+ (testing "a declared error passes through"
+ (is (= {:class :unrecoverable :code ::out-of-ink :user-message "Out of ink."}
+ (:error (invoke "write" {:declared true}))))))
+
+(deftest ^:parallel a-later-turn-replays-only-that-an-undeclared-failure-happened-test
+ (testing "the live instruction not to retry is for the turn that failed, so a later turn replays one neutral sentence"
+ (is (= "This call failed with an unexpected error."
+ (tools.runtime/replay-text (invoke "crash" {})))))
+ (testing "every other failure replays the text the model read live"
+ (doseq [[tool-name args] [["boom" {:id 9}] ["give_up" {}] ["write" {}] ["jvm_failure" {:kind "overflow"}]]]
+ (let [outcome (invoke tool-name args)]
+ (is (= (:output outcome) (tools.runtime/replay-text outcome)) tool-name)))))
+
(deftest ^:parallel a-wrong-handler-result-is-our-bug-not-the-models-test
(is (= {:class :unrecoverable :code :internal} (:error (invoke "bad_shape" {})))))
@@ -396,16 +436,22 @@
(is (= {:unknown-tool :validation
:invalid-arguments :validation
:scope-denied :unrecoverable
- :internal :unrecoverable}
+ :internal :unrecoverable
+ :jvm-error :unrecoverable
+ :interrupted :unrecoverable
+ :unconfirmed-write :unrecoverable}
tools.runtime/runtime-codes))
(testing "and every one of them is reachable through invoke"
;; The scoped case needs a ctx that grants nothing; the rest run under the default
;; unrestricted one.
- (is (= #{:unknown-tool :invalid-arguments :scope-denied :internal}
+ (is (= #{:unknown-tool :invalid-arguments :scope-denied :internal :jvm-error :interrupted :unconfirmed-write}
(set (keep (fn [[tool-name args overrides]]
(get-in (tools.runtime/invoke entries (ctx overrides) tool-name args)
[:error :code]))
[["nope" {}]
["happy" {:m 1}]
["scoped" {} {:user-scope #{}}]
- ["crash" {}]])))))))
+ ["crash" {}]
+ ["jvm_failure" {:kind "overflow"}]
+ ["jvm_failure" {:kind "interrupt"}]
+ ["write" {}]])))))))
diff --git a/test/metabase/metabot/tools/save_entity_test.clj b/test/metabase/metabot/tools/save_entity_test.clj
index ece0da47b5e..300366ced43 100644
--- a/test/metabase/metabot/tools/save_entity_test.clj
+++ b/test/metabase/metabot/tools/save_entity_test.clj
@@ -431,9 +431,13 @@
(mt/with-current-user (mt/user->id :crowberto)
(mt/with-dynamic-fn-redefs [queries/create-card!
(fn [& _] (throw (ex-info "Private save failure." {:status-code 500})))]
- (is (= {:error {:class :unrecoverable :code :internal}
- :output "This call failed and the user was shown the error (:internal). Don't retry it."}
- (invoke-save-entity (assoc chart-args :destination {:target_type "collection"})))))))
+ (testing "the card may have been saved before the failure, so the turn ends and the user is told to check"
+ (is (= {:output "This call failed and the user was shown the error (:unconfirmed-write). Don't retry it."
+ :error {:class :unrecoverable
+ :code :unconfirmed-write
+ :user-message (str "Something went wrong. The change may have been saved, "
+ "so check before you try again.")}}
+ (invoke-save-entity (assoc chart-args :destination {:target_type "collection"}))))))))
(deftest sensitive-query-test
(let [provider (mt/metadata-provider)
diff --git a/test/metabase/metabot/tools_test.clj b/test/metabase/metabot/tools_test.clj
index 0eee78f8fb8..971c2e07d3b 100644
--- a/test/metabase/metabot/tools_test.clj
+++ b/test/metabase/metabot/tools_test.clj
@@ -824,7 +824,7 @@
(is (= :metabase.metabot.tools.recoverable.pipeline/unknown-table code))
(is (str/includes? text "No table found matching portable FK"))
(is (str/includes? text "metabase://database/<numeric id>/tables"))))
- (testing "An exception without a declared `:error` code ends the turn, even if `:agent-error?` is true."
+ (testing "An exception without a declared `:error` code is `:internal`, even if `:agent-error?` is true."
(doseq [[what e] {"stamped by the pipeline, no :error code"
(ex-info "No matching clause: :metabase.lib.schema/bogus" {:agent-error? true})