Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Make SQL guardrail skills always-on (#76176)

Mihael Konjevićb907ae761d84
diff --git a/src/metabase/metabot/agent/core.clj b/src/metabase/metabot/agent/core.clj
index ef90d437ed4..b695aadd3ec 100644
--- a/src/metabase/metabot/agent/core.clj
+++ b/src/metabase/metabot/agent/core.clj
@@ -176,27 +176,47 @@
[parts]
(some #(= (:type %) :tool-input) parts))
-(defn- has-final-response?
- "Check if any tool result signals a final response."
- [parts]
- (some #(and (= (:type %) :tool-output)
- (get-in % [:result :final-response?]))
- parts))
+(defn- successful-tool-output?
+ "Whether a `:tool-output` part represents a successful call.
+ Answer-producing tools attach `:structured-output` when they produce a result (a query, a chart
+ draft, a clarification question), and return just an `:output` error string on validation/exception
+ failures — so its presence is the success signal."
+ [part]
+ (and (= (:type part) :tool-output)
+ (some? (get-in part [:result :structured-output]))))
+
+(defn- terminal-tool-call?
+ "Whether `parts` contain a **successful** call to one of the profile's `terminal-tools` (a set of
+ tool-name strings). This lets a profile end the turn as soon as it produces its answer (e.g. the
+ `:sql` profile after a successful `edit_sql_query`, or `ask_for_sql_clarification`) instead of
+ forcing the model to keep emitting tool calls under `:required-tool-call?`.
+
+ Terminality is a per-profile decision — the same tool is non-terminal in profiles that don't list
+ it. A *failed* terminal-tool call does not end the turn, so the model can still self-correct."
+ [terminal-tools parts]
+ (boolean
+ (when (seq terminal-tools)
+ (let [success-ids (into #{} (comp (filter successful-tool-output?) (map :id)) parts)]
+ (some (fn [p]
+ (and (= (:type p) :tool-input)
+ (contains? terminal-tools (:function p))
+ (contains? success-ids (:id p))))
+ parts)))))
(defn- should-continue?
"Determine if agent should continue iterating."
- [iteration max-iterations parts]
+ [iteration max-iterations terminal-tools parts]
(and (< iteration max-iterations)
(has-tool-calls? parts)
- (not (has-final-response? parts))))
+ (not (terminal-tool-call? terminal-tools parts))))
(defn- finish-reason
"Determine why the agent loop stopped."
- [iteration max-iterations parts]
+ [iteration max-iterations terminal-tools parts]
(cond
- (>= iteration max-iterations) :max-iterations
- (has-final-response? parts) :final-response
- :else :stop))
+ (>= iteration max-iterations) :max-iterations
+ (terminal-tool-call? terminal-tools parts) :terminal-tool
+ :else :stop))
;;; Call LLM
(defn- part->invert-links-key
@@ -478,6 +498,7 @@
:iteration iteration}
(let [{:keys [profile tools context memory-atom tracking-opts]} agent
max-iter (:max-iterations profile 10)
+ terminal-tools (set (:terminal-tools profile))
tracking-opts (assoc tracking-opts :iteration iteration)
memory @memory-atom
parts-atom (atom [])
@@ -510,14 +531,14 @@
(reduced? result')
(assoc loop-state :status :reduced :result @result')
- (should-continue? iteration max-iter parts)
+ (should-continue? iteration max-iter terminal-tools parts)
(assoc loop-state :result result' :iteration (inc iteration))
:else
(do (log/info "Agent loop complete"
{:iterations iteration
;; TODO: decide if we want this reason to float up to frontend
- :reason (finish-reason iteration max-iter parts)})
+ :reason (finish-reason iteration max-iter terminal-tools parts)})
(assoc loop-state
:status :done
:result (rf result' (final-state-part @memory-atom))))))))))
diff --git a/src/metabase/metabot/agent/profiles.clj b/src/metabase/metabot/agent/profiles.clj
index 276a345eaec..486c960317d 100644
--- a/src/metabase/metabot/agent/profiles.clj
+++ b/src/metabase/metabot/agent/profiles.clj
@@ -53,21 +53,40 @@
- :max-iterations - Maximum agent loop iterations
- :temperature - LLM temperature setting
- :tools - Vector of tool vars (e.g. #'tools/search-tool)
+ - :always-on-skills - Optional vector of skill ids (keywords) whose bodies are inlined into this
+ profile's system prompt instead of being loaded on demand via `load_skill`. Always-on is a
+ per-profile decision: the same skill can be inlined here and on-demand elsewhere.
+ - :terminal-tools - Optional set of tool-name strings whose **successful** call ends the agent
+ turn for this profile. Lets a `:required-tool-call?` profile stop as soon as it produces its
+ answer (e.g. `:sql` after `edit_sql_query`) instead of being forced to keep calling tools.
+ Terminality is per-profile: the same tool is non-terminal in profiles that don't list it.
- Tool vars are validated at registration time to ensure they have required metadata."
+ Tool vars are validated at registration time to ensure they have required metadata; any
+ `:always-on-skills` are validated to refer to registered skills, and any `:terminal-tools` to
+ refer to tools the profile actually exposes."
[profile :- [:map
[:name :keyword]
[:prompt-template :string]
[:max-iterations :int]
[:temperature :float]
- [:tools [:vector :any]]]]
- (let [tool-vars (:tools profile)]
+ [:tools [:vector :any]]
+ [:always-on-skills {:optional true} [:vector :keyword]]
+ [:terminal-tools {:optional true} [:set :string]]]]
+ (let [tool-vars (:tools profile)
+ tool-name-seq (map #(:tool-name (meta %)) tool-vars)
+ tool-names (set tool-name-seq)]
(doseq [tool-var tool-vars]
(validate-tool-var! tool-var))
- (when-not (apply distinct? (map #(:tool-name (meta %)) tool-vars))
- (let [dups (->> (frequencies (map #(:tool-name (meta %)) tool-vars))
+ (when-not (apply distinct? tool-name-seq)
+ (let [dups (->> (frequencies tool-name-seq)
(filter (fn [[_ cnt]] (< 1 cnt))))]
- (throw (ex-info "Duplicate tool names in profile" {:tool-names (map first dups)})))))
+ (throw (ex-info "Duplicate tool names in profile" {:tool-names (map first dups)}))))
+ (when-let [unknown (seq (remove skills/get-skill (:always-on-skills profile)))]
+ (throw (ex-info "Profile references unknown always-on skill ids"
+ {:profile (:name profile) :unknown-skill-ids unknown})))
+ (when-let [unknown (seq (remove tool-names (:terminal-tools profile)))]
+ (throw (ex-info "Profile lists terminal tools it does not expose"
+ {:profile (:name profile) :unknown-terminal-tools unknown}))))
(swap! *profiles assoc (:name profile) profile))
(register-profile!
@@ -125,6 +144,20 @@
:max-iterations 20
:temperature 0.3
:required-tool-call? true
+ ;; The SQL editor is a focused, tool-heavy flow: this guidance is relevant on essentially every
+ ;; turn, so inline it rather than make the model spend iterations loading it. Other profiles that
+ ;; share these tools (e.g. :internal) keep them on-demand by not listing them here.
+ :always-on-skills [:read-resource
+ :create-sql-query
+ :edit-sql-query
+ :replace-sql-query
+ :ask-for-sql-clarification]
+ ;; Delivering SQL via one of these tools — or asking the user to clarify — IS the answer here, so a
+ ;; successful call ends the turn rather than forcing more tool calls under :required-tool-call?.
+ ;; Failed calls don't terminate, so the model can still self-correct. The same tools stay
+ ;; non-terminal in other profiles.
+ :terminal-tools #{"create_sql_query" "edit_sql_query" "replace_sql_query"
+ "ask_for_sql_clarification"}
:tools [#'tools/sql-search-tool
#'tools/read-resource-tool
#'tools/create-sql-query-code-edit-tool
@@ -150,6 +183,9 @@
:max-iterations 10
:temperature 0.3
:required-tool-call? true
+ ;; Producing a chart draft is the answer; a successful construct ends the turn (schema collection
+ ;; is a non-terminal preparatory step). Failed constructs don't terminate, so the model retries.
+ :terminal-tools #{"document_construct_model_chart" "document_construct_sql_chart"}
:tools [#'tools/list-available-data-sources-tool
#'tools/list-available-fields-tool
#'tools/get-field-values-tool
@@ -218,11 +254,16 @@
`load_skill` tool is injected so the agent can pull skill bodies on demand."
[profile-id capabilities]
(when-let [profile (get-profile profile-id)]
- (let [base (-> profile
- :tools
- (filter-by-capabilities capabilities)
- filter-by-scope
- tool-map)]
+ (let [base (-> profile
+ :tools
+ (filter-by-capabilities capabilities)
+ filter-by-scope
+ tool-map)
+ manifest (skills/build-skill-manifest profile (keys base) capabilities)]
(cond-> base
- (seq (:catalog (skills/build-skill-manifest profile (keys base) capabilities)))
+ ;; Register load_skill whenever the profile has ANY skills — on-demand (catalog) or
+ ;; always-on. Always-on bodies are inlined, but their presence still implies a
+ ;; skill-bearing, possibly dialect-capable profile whose `dialect-preload-parts` emit a
+ ;; synthetic `load_skill` call that must resolve to a registered tool.
+ (or (seq (:catalog manifest)) (seq (:always-on manifest)))
(assoc "load_skill" #'tools/load-skill-tool)))))
diff --git a/src/metabase/metabot/agent/prompts.clj b/src/metabase/metabot/agent/prompts.clj
index c1712b88a26..e132466c225 100644
--- a/src/metabase/metabot/agent/prompts.clj
+++ b/src/metabase/metabot/agent/prompts.clj
@@ -181,7 +181,11 @@
:first_day_of_week first-day-of-week
:sql_dialect sql-dialect
:sql_dialect_loaded (some? (skills/dialect-skill sql-dialect))
- :skill_catalog catalog
+ ;; `not-empty` so an empty catalog is nil (falsy) — Selmer treats
+ ;; an empty vector as truthy, which would render the "# Available
+ ;; skills … load the skill(s) you need" header with nothing to
+ ;; load, nudging the model into pointless `load_skill` calls.
+ :skill_catalog (not-empty catalog)
:skill_always_on (mapv :body always-on)
:viewing_context viewing-context
:recent_views recent-views
diff --git a/src/metabase/metabot/skills.clj b/src/metabase/metabot/skills.clj
index 8f521aa8ab2..859efb96825 100644
--- a/src/metabase/metabot/skills.clj
+++ b/src/metabase/metabot/skills.clj
@@ -43,7 +43,6 @@
[:profiles {:optional true} [:maybe [:vector :keyword]]]
[:capabilities {:optional true} [:vector :keyword]]
[:scope {:optional true} :keyword]
- [:always-on {:optional true} :boolean]
[:priority {:optional true} :int]
[:dialect {:optional true} :string]])
@@ -68,8 +67,7 @@
(:profiles skill) (update :profiles #(mapv keyword %))
(:capabilities skill) (update :capabilities #(mapv keyword %))
(:scope skill) (update :scope keyword)
- true (update :priority (fnil int 50))
- true (update :always-on boolean)))
+ true (update :priority (fnil int 50))))
(defn- validate-skill!
[skill]
@@ -244,20 +242,31 @@
(defn build-skill-manifest
"Build the skill manifest for a profile + active tools, gated by `capabilities` and the user's scope.
Returns {:always-on [skill-map…] :catalog [{:id :title :description}…]} sorted by descending priority
- then id."
+ then id.
+
+ Whether a skill is *always-on* (body inlined in the system prompt) vs *on-demand* (advertised in the
+ catalog, loaded via `load_skill`) is decided **per profile**, by the profile's `:always-on-skills`
+ set — not by the skill itself — so the same skill can be inlined for one profile and on-demand for
+ another (e.g. the SQL skills are always-on for `:sql` but on-demand for `:internal`)."
[profile active-tool-names capabilities]
- (let [cap-set (capabilities/capability-set capabilities)
- visible (->> (skills-for-profile profile active-tool-names)
- (filter #(skill-visible? % cap-set))
- (sort-by (juxt (comp - :priority) (comp str :id))))]
- (record-loadable-skill-ids! visible)
- {:always-on (filter :always-on visible)
- :catalog (->> visible
- (remove :always-on)
- (mapv (fn [s]
- {:id (name (:id s))
- :title (:title s)
- :description (:description s)})))}))
+ (let [cap-set (capabilities/capability-set capabilities)
+ always-on-ids (set (:always-on-skills profile))
+ always-on? (comp always-on-ids :id)
+ visible (->> (skills-for-profile profile active-tool-names)
+ (filter #(skill-visible? % cap-set))
+ (sort-by (juxt (comp - :priority) (comp str :id))))
+ ;; Always-on skill bodies are already inlined in the system prompt, so they are neither
+ ;; advertised in the catalog nor loadable on demand — only the on-demand skills are.
+ ;; Recording only the loadable ids also stops `load_skill` from re-fetching an always-on
+ ;; skill (a wasted iteration the model would otherwise spend).
+ loadable (remove always-on? visible)]
+ (record-loadable-skill-ids! loadable)
+ {:always-on (filter always-on? visible)
+ :catalog (mapv (fn [s]
+ {:id (name (:id s))
+ :title (:title s)
+ :description (:description s)})
+ loadable)}))
;;; Dialect preload
@@ -292,11 +301,11 @@
Dialect context only arises in SQL-editor sessions, so its presence is itself the gate.
Invariant: the emitted pair references the `load_skill` tool, which
- [[metabase.metabot.agent.profiles/get-tools-for-profile]] registers whenever the skill catalog is
- non-empty.
- Any profile that can reach SQL-editor/dialect context also exposes query skills (the cross-cutting
- `read-resource` skill alone keeps the catalog non-empty), so `load_skill` is always registered when
- this returns a non-empty preload."
+ [[metabase.metabot.agent.profiles/get-tools-for-profile]] registers whenever the skill manifest is
+ non-empty — i.e. the profile has any catalog or always-on skill.
+ Any profile that can reach SQL-editor/dialect context also exposes query skills (e.g. the SQL
+ profile inlines the `read-resource` skill via its `:always-on-skills`), so the manifest is non-empty
+ and `load_skill` is always registered when this returns a non-empty preload."
[engine]
(or (when-let [skill (dialect-skill engine)]
(let [skill-id (:id skill)
diff --git a/src/metabase/metabot/tools/clarification.clj b/src/metabase/metabot/tools/clarification.clj
index 8ec2416953f..9230aab7604 100644
--- a/src/metabase/metabot/tools/clarification.clj
+++ b/src/metabase/metabot/tools/clarification.clj
@@ -12,8 +12,9 @@
"Ask the user for clarification about their SQL query request.
This tool is used when the agent needs more information from the user
- to properly complete their SQL request. It returns a final response
- that stops the agent loop and waits for user input.
+ to properly complete their SQL request. Profiles that list
+ `ask_for_sql_clarification` in their `:terminal-tools` (e.g. `:sql`) end the
+ turn on a successful call and wait for user input.
Parameters:
- question: The clarification question to ask the user
@@ -21,14 +22,12 @@
Returns map with:
- :structured-output - The question and options for the LLM
- - :final-response? - Signals that the agent should stop and wait for user input
- :instructions - Instructions for the LLM"
[{:keys [question options]}]
(log/info "Asking for SQL clarification" {:question question
:option-count (count options)})
{:structured-output {:question question
:options (or options [])}
- :final-response? true
:instructions "The clarification question has been presented to the user. Stop and wait for their response before continuing."})
(defn- format-clarification-output
diff --git a/src/metabase/metabot/tools/document.clj b/src/metabase/metabot/tools/document.clj
index c4f0e8a027a..53e599541b0 100644
--- a/src/metabase/metabot/tools/document.clj
+++ b/src/metabase/metabot/tools/document.clj
@@ -171,8 +171,7 @@
:query query
:result-type :chart-draft}]
{:output "Draft chart payload generated from SQL query."
- :structured-output structured
- :final-response? true}))))
+ :structured-output structured}))))
(catch Exception e
(log/error e "Error constructing SQL chart draft")
(if (:agent-error? (ex-data e))
@@ -216,8 +215,7 @@
:chart_type chart-type
:query_id query-id
:query dataset-query
- :result-type :chart-draft}
- :final-response? true}
+ :result-type :chart-draft}}
;; Preserve tool error messaging from construct_notebook_query path.
(or result
{:output "Failed to construct model chart draft."})))
diff --git a/test/metabase/metabot/agent/core_test.clj b/test/metabase/metabot/agent/core_test.clj
index d97ffeb8584..00a3b35a0fd 100644
--- a/test/metabase/metabot/agent/core_test.clj
+++ b/test/metabase/metabot/agent/core_test.clj
@@ -34,22 +34,48 @@
{:type :tool-input :id "t1"}]))))
(deftest should-continue-test
- (let [max-iter 3]
+ (let [max-iter 3
+ no-term #{}]
(testing "continues when iteration < max and has tool calls"
- (is (#'agent/should-continue? 0 max-iter [{:type :tool-input}]))
- (is (#'agent/should-continue? 1 max-iter [{:type :tool-input}])))
+ (is (#'agent/should-continue? 0 max-iter no-term [{:type :tool-input}]))
+ (is (#'agent/should-continue? 1 max-iter no-term [{:type :tool-input}])))
(testing "continues when text AND tool calls present (LLM thinking aloud)"
- (is (#'agent/should-continue? 0 max-iter [{:type :tool-input}
- {:type :text}]))
- (is (#'agent/should-continue? 0 max-iter [{:type :text}
- {:type :tool-input}])))
+ (is (#'agent/should-continue? 0 max-iter no-term [{:type :tool-input}
+ {:type :text}]))
+ (is (#'agent/should-continue? 0 max-iter no-term [{:type :text}
+ {:type :tool-input}])))
(testing "stops at max iterations (1-based: iteration >= max means done)"
- (is (not (#'agent/should-continue? 3 max-iter [{:type :tool-input}])))
- (is (not (#'agent/should-continue? 4 max-iter [{:type :tool-input}]))))
+ (is (not (#'agent/should-continue? 3 max-iter no-term [{:type :tool-input}])))
+ (is (not (#'agent/should-continue? 4 max-iter no-term [{:type :tool-input}]))))
(testing "stops when no tool calls (text-only is final answer)"
- (is (not (#'agent/should-continue? 0 max-iter [{:type :text}])))
- (is (not (#'agent/should-continue? 0 max-iter [{:type :usage}])))
- (is (not (#'agent/should-continue? 0 max-iter []))))))
+ (is (not (#'agent/should-continue? 0 max-iter no-term [{:type :text}])))
+ (is (not (#'agent/should-continue? 0 max-iter no-term [{:type :usage}])))
+ (is (not (#'agent/should-continue? 0 max-iter no-term []))))))
+
+(deftest terminal-tool-call-test
+ (let [terminal #{"edit_sql_query" "create_sql_query" "replace_sql_query"}
+ success [{:type :tool-input :id "a" :function "edit_sql_query"}
+ {:type :tool-output :id "a" :result {:output "ok"
+ :structured-output {:result-type :query
+ :query-id "q1"}}}]
+ failure [{:type :tool-input :id "b" :function "edit_sql_query"}
+ {:type :tool-output :id "b" :result {:output "validation error"
+ :instructions "fix it"}}]
+ read [{:type :tool-input :id "c" :function "read_resource"}
+ {:type :tool-output :id "c" :result {:output "<fields/>"}}]]
+ (testing "a successful terminal-tool call ends the turn"
+ (is (#'agent/terminal-tool-call? terminal success))
+ (is (not (#'agent/should-continue? 0 20 terminal success))
+ "should-continue? is false right after a successful edit (the 10402 fix)"))
+ (testing "a FAILED terminal-tool call does not end the turn (model can self-correct)"
+ (is (not (#'agent/terminal-tool-call? terminal failure)))
+ (is (#'agent/should-continue? 0 20 terminal failure)))
+ (testing "a non-terminal tool (read_resource) does not end the turn"
+ (is (not (#'agent/terminal-tool-call? terminal read))))
+ (testing "terminality is per-profile: an empty terminal set never ends the turn"
+ (is (not (#'agent/terminal-tool-call? #{} success))))
+ (testing "finish-reason reports :terminal-tool"
+ (is (= :terminal-tool (#'agent/finish-reason 0 20 terminal success))))))
(deftest run-agent-loop-with-mock-test
(mt/as-admin
diff --git a/test/metabase/metabot/agent/profiles_test.clj b/test/metabase/metabot/agent/profiles_test.clj
index ca5c246317e..9bc90cff472 100644
--- a/test/metabase/metabot/agent/profiles_test.clj
+++ b/test/metabase/metabot/agent/profiles_test.clj
@@ -4,6 +4,7 @@
[metabase.api-scope.core :as api-scope]
[metabase.metabot.agent.profiles :as profiles]
[metabase.metabot.scope :as scope]
+ [metabase.metabot.tools :as tools]
[metabase.metabot.tools.transforms :as tools.transforms]
[metabase.premium-features.core :as premium-features]
[metabase.test :as mt]))
@@ -215,3 +216,29 @@
(is (= #{}
(set (#'profiles/filter-by-capabilities transform-tools
["permission:write_transforms"]))))))))
+
+(deftest terminal-tools-test
+ (testing "the :sql profile marks its SQL write tools AND clarification terminal"
+ (is (= #{"create_sql_query" "edit_sql_query" "replace_sql_query" "ask_for_sql_clarification"}
+ (:terminal-tools (profiles/get-profile :sql)))))
+ (testing "the document profile ends the turn on a constructed chart, not on schema collection"
+ (is (= #{"document_construct_model_chart" "document_construct_sql_chart"}
+ (:terminal-tools (profiles/get-profile :document-generate-content)))))
+ (testing "terminality is per-profile — profiles that share these tools don't inherit it"
+ (is (nil? (:terminal-tools (profiles/get-profile :internal))))
+ (is (nil? (:terminal-tools (profiles/get-profile :nlq))))))
+
+(deftest register-profile-validation-test
+ (let [base {:name :scratch
+ :prompt-template "internal.selmer"
+ :max-iterations 10
+ :temperature 0.3
+ :tools [#'tools/read-resource-tool]}]
+ (testing "rejects :always-on-skills that don't resolve to a registered skill"
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"unknown always-on skill"
+ (#'profiles/register-profile!
+ (assoc base :always-on-skills [:no-such-skill])))))
+ (testing "rejects :terminal-tools the profile does not expose"
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"terminal tools it does not expose"
+ (#'profiles/register-profile!
+ (assoc base :terminal-tools #{"nonexistent_tool"})))))))
diff --git a/test/metabase/metabot/skills_test.clj b/test/metabase/metabot/skills_test.clj
index def21a2ff5a..29e1af3935a 100644
--- a/test/metabase/metabot/skills_test.clj
+++ b/test/metabase/metabot/skills_test.clj
@@ -2,6 +2,7 @@
(:require
[clojure.string :as str]
[clojure.test :refer [deftest is testing]]
+ [metabase.metabot.agent.prompts :as prompts]
[metabase.metabot.scope :as scope]
[metabase.metabot.skills :as skills]
[metabase.test :as mt]))
@@ -85,30 +86,90 @@
(is (empty? (filter :dialect (skills/skills-for-profile profile ["construct_notebook_query"])))))))
(deftest ^:parallel build-skill-manifest-test
- (let [profile {:name :internal}
+ ;; :internal declares no :always-on-skills, so every relevant skill is on-demand (catalog).
+ (let [profile {:name :internal}
manifest (skills/build-skill-manifest
profile
["construct_notebook_query" "read_resource"]
[])]
- (testing "catalog lists available skills with string ids"
+ (testing "catalog lists available on-demand skills with string ids"
(let [ids (map :id (:catalog manifest))]
(is (every? string? ids))
(is (some #{"construct-notebook-query-core"} ids))
- (is (some #{"read-resource"} ids))))
+ (is (some #{"read-resource"} ids)
+ "without :always-on-skills, read-resource is on-demand for this profile")))
(testing "catalog entries are display-only (id/title/description), no body"
(is (every? #(= #{:id :title :description} (set (keys %))) (:catalog manifest))))
(testing "catalog is sorted by descending priority then id"
;; core has priority 60, the rest 50, operators 40 -> core first, operators last
(is (= "construct-notebook-query-core" (:id (first (:catalog manifest))))))
- (testing "no always-on skills in this migration"
+ (testing "no always-on skills when the profile declares none"
(is (empty? (:always-on manifest))))))
+(deftest ^:parallel build-skill-manifest-always-on-is-per-profile-test
+ (testing "a skill listed in the profile's :always-on-skills is inlined (body) and removed from
+ the catalog, while the same skill is on-demand for a profile that doesn't list it"
+ (let [tools ["read_resource"]
+ inlined (skills/build-skill-manifest
+ {:name :sql :always-on-skills [:read-resource]} tools [])
+ on-demand (skills/build-skill-manifest
+ {:name :internal} tools [])]
+ (testing "profile that opts in -> always-on with body, absent from catalog"
+ (is (= ["read-resource"] (map (comp name :id) (:always-on inlined))))
+ (is (every? :body (:always-on inlined)))
+ (is (not (some #{"read-resource"} (map :id (:catalog inlined))))))
+ (testing "profile that opts out -> on-demand in catalog, not always-on"
+ (is (empty? (:always-on on-demand)))
+ (is (some #{"read-resource"} (map :id (:catalog on-demand))))))))
+
+(deftest ^:parallel always-on-skill-body-is-in-system-prompt-test
+ ;; A "marker" line lifted verbatim from the read-resource skill body. If the skill is inlined,
+ ;; this text appears in the rendered system prompt; if it is on-demand, only its one-line catalog
+ ;; description appears (which does not contain this phrase).
+ (let [marker "Drill, don't re-search"
+ tools {"read_resource" nil}
+ render (fn [profile]
+ (binding [scope/*current-user-metabot-permissions*
+ {:permission/metabot-sql-generation :yes
+ :permission/metabot-nlq :no
+ :permission/metabot-other-tools :no}]
+ (prompts/build-system-message-content
+ (assoc profile :prompt-template "sql-querying-only.selmer")
+ {:current_time "2026-06-19T12:00:00Z"}
+ tools
+ [])))]
+ (testing "read-resource body IS present when the profile marks it always-on"
+ (let [rendered (render {:name :sql :always-on-skills [:read-resource]})]
+ (is (str/includes? rendered marker)
+ "always-on skill body must be inlined into the system prompt")
+ (is (str/includes? rendered (:body (skills/get-skill :read-resource)))
+ "the full skill body is inlined, not just a fragment")
+ (is (not (str/includes? rendered "# Available skills"))
+ "an empty catalog must NOT render the load_skill header (would invite pointless loads)")))
+ (testing "read-resource body is ABSENT when the profile leaves it on-demand"
+ (let [rendered (render {:name :sql})]
+ (is (not (str/includes? rendered marker))
+ "an on-demand skill's body must not leak into the prompt prefix")
+ (is (str/includes? rendered "# Available skills")
+ "a non-empty catalog renders the load_skill header")
+ (is (str/includes? rendered "read-resource")
+ "the on-demand skill still appears as a one-line catalog entry")))))
+
(deftest ^:parallel build-skill-manifest-records-loadable-skill-ids-test
(testing "building the manifest records the request's loadable skill ids"
(binding [scope/*current-loadable-skill-ids* (atom #{})]
(skills/build-skill-manifest {:name :internal} ["construct_notebook_query"] [])
(is (contains? @scope/*current-loadable-skill-ids* :construct-notebook-query-core))
- (is (not (contains? @scope/*current-loadable-skill-ids* :read-resource))))))
+ (is (not (contains? @scope/*current-loadable-skill-ids* :read-resource)))))
+ (testing "a profile's always-on skills are NOT loadable — their bodies are already inlined, so
+ load_skill must not re-fetch them (a wasted iteration)"
+ (binding [scope/*current-loadable-skill-ids* (atom #{})]
+ (skills/build-skill-manifest
+ {:name :sql :always-on-skills [:read-resource]} ["read_resource"] [])
+ (is (not (contains? @scope/*current-loadable-skill-ids* :read-resource))
+ "an always-on skill must be excluded from the loadable set")
+ (is (not (skills/skill-loadable? (skills/get-skill :read-resource)))
+ "skill-loadable? rejects an always-on skill once a manifest is tracked"))))
(deftest ^:parallel dialect-preload-parts-test
(testing "resolves a synthetic load_skill pair when the dialect is known"
diff --git a/test/metabase/metabot/tools/clarification_test.clj b/test/metabase/metabot/tools/clarification_test.clj
index ba4bde17485..13438c75def 100644
--- a/test/metabase/metabot/tools/clarification_test.clj
+++ b/test/metabase/metabot/tools/clarification_test.clj
@@ -5,10 +5,10 @@
[metabase.metabot.tools.clarification :as ask-clarification]))
(deftest ask-for-sql-clarification-test
- (testing "returns final-response? flag"
+ (testing "returns structured-output (the success signal that lets a profile end the turn)"
(let [result (ask-clarification/ask-for-sql-clarification {:question "What table should I use?"})]
(is (map? result))
- (is (true? (:final-response? result)))))
+ (is (some? (:structured-output result)))))
(testing "returns question in structured output"
(let [result (ask-clarification/ask-for-sql-clarification {:question "What columns do you need?"})]
(is (= "What columns do you need?" (get-in result [:structured-output :question])))))
@@ -26,10 +26,13 @@
(is (string? (:instructions result)))
(is (str/includes? (:instructions result) "wait")))))
-(deftest final-response-behavior-test
- (testing "final-response? signals agent should stop"
- ;; This is a contract test - the agent loop checks for :final-response? true
- ;; to determine if it should stop processing
+(deftest terminal-via-structured-output-test
+ (testing "the tool produces :structured-output so a profile with it in :terminal-tools stops"
+ ;; Contract: termination is profile-driven (see metabase.metabot.agent.profiles/:sql
+ ;; :terminal-tools). The agent loop ends the turn on a *successful* terminal-tool call, and a
+ ;; successful clarification is signalled by the presence of :structured-output.
(let [result (ask-clarification/ask-for-sql-clarification {:question "Test?"})]
- (is (true? (:final-response? result))
- "ask_for_sql_clarification must return :final-response? true to stop the agent loop"))))
+ (is (some? (:structured-output result))
+ "ask_for_sql_clarification must return :structured-output so the agent loop can stop")
+ (is (not (contains? result :final-response?))
+ "the tool-level :final-response? flag has been removed in favour of :terminal-tools"))))
diff --git a/test/metabase/metabot/tools/document_test.clj b/test/metabase/metabot/tools/document_test.clj
index 073d0474631..9f7ff96aa2e 100644
--- a/test/metabase/metabot/tools/document_test.clj
+++ b/test/metabase/metabot/tools/document_test.clj
@@ -57,7 +57,8 @@
:sql "SELECT * FROM test"
:viz_settings {:chart_type "bar"}})
structured (:structured-output result)]
- (is (true? (:final-response? result)))
+ (is (some? structured)
+ "structured-output present -> a successful, terminal-eligible call")
(is (= "document_construct_sql_chart" (:tool structured)))
(is (= "Test Name" (:name structured)))
(is (= "Test Desc" (:description structured)))
@@ -85,8 +86,8 @@
:approach "Test Approach"
:sql "SELECT FROM test"
:viz_settings {:chart_type "bar"}})]
- (is (nil? (:final-response? result)))
- (is (nil? (:structured-output result)))
+ (is (nil? (:structured-output result))
+ "no structured-output -> a failure, which must NOT terminate the turn")
(is (re-find #"SQL chart draft generation failed" (:output result)))
(is (re-find #"syntax error near FROM" (:output result)))))))
@@ -111,8 +112,8 @@
:approach "Test Approach"
:sql "SELECT * FROM missing_table"
:viz_settings {:chart_type "bar"}})]
- (is (nil? (:final-response? result)))
- (is (nil? (:structured-output result)))
+ (is (nil? (:structured-output result))
+ "no structured-output -> a failure, which must NOT terminate the turn")
(is (re-find #"could not be processed by Metabase" (:output result)))
(is (re-find #"missing_table" (:output result)))))))
@@ -131,7 +132,8 @@
:query ""
:viz_settings {:chart_type "bar"}})
structured (:structured-output result)]
- (is (true? (:final-response? result)))
+ (is (some? structured)
+ "structured-output present -> a successful, terminal-eligible call")
(is (= "document_construct_model_chart" (:tool structured)))
(is (= "Test Name" (:name structured)))
(is (= "Test Desc" (:description structured)))