The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Make SQL guardrail skills always-on (#76176)
Mihael Konjevićb907ae761d84
diff --git a/src/metabase/metabot/agent/core.clj b/src/metabase/metabot/agent/core.cljindex ef90d437ed4..b695aadd3ec 100644--- a/src/metabase/metabot/agent/core.clj+++ b/src/metabase/metabot/agent/core.clj@@ -176,27 +176,47 @@[parts](some #(= (:type %) :tool-input) parts))-(defn- has-final-response?- "Check if any tool result signals a final response."- [parts]- (some #(and (= (:type %) :tool-output)- (get-in % [:result :final-response?]))- parts))+(defn- successful-tool-output?+ "Whether a `:tool-output` part represents a successful call.+ Answer-producing tools attach `:structured-output` when they produce a result (a query, a chart+ draft, a clarification question), and return just an `:output` error string on validation/exception+ failures — so its presence is the success signal."+ [part]+ (and (= (:type part) :tool-output)+ (some? (get-in part [:result :structured-output]))))++(defn- terminal-tool-call?+ "Whether `parts` contain a **successful** call to one of the profile's `terminal-tools` (a set of+ tool-name strings). This lets a profile end the turn as soon as it produces its answer (e.g. the+ `:sql` profile after a successful `edit_sql_query`, or `ask_for_sql_clarification`) instead of+ forcing the model to keep emitting tool calls under `:required-tool-call?`.++ Terminality is a per-profile decision — the same tool is non-terminal in profiles that don't list+ it. A *failed* terminal-tool call does not end the turn, so the model can still self-correct."+ [terminal-tools parts]+ (boolean+ (when (seq terminal-tools)+ (let [success-ids (into #{} (comp (filter successful-tool-output?) (map :id)) parts)]+ (some (fn [p]+ (and (= (:type p) :tool-input)+ (contains? terminal-tools (:function p))+ (contains? success-ids (:id p))))+ parts)))))(defn- should-continue?"Determine if agent should continue iterating."- [iteration max-iterations parts]+ [iteration max-iterations terminal-tools parts](and (< iteration max-iterations)(has-tool-calls? parts)- (not (has-final-response? parts))))+ (not (terminal-tool-call? terminal-tools parts))))(defn- finish-reason"Determine why the agent loop stopped."- [iteration max-iterations parts]+ [iteration max-iterations terminal-tools parts](cond- (>= iteration max-iterations) :max-iterations- (has-final-response? parts) :final-response- :else :stop))+ (>= iteration max-iterations) :max-iterations+ (terminal-tool-call? terminal-tools parts) :terminal-tool+ :else :stop));;; Call LLM(defn- part->invert-links-key@@ -478,6 +498,7 @@:iteration iteration}(let [{:keys [profile tools context memory-atom tracking-opts]} agentmax-iter (:max-iterations profile 10)+ terminal-tools (set (:terminal-tools profile))tracking-opts (assoc tracking-opts :iteration iteration)memory @memory-atomparts-atom (atom [])@@ -510,14 +531,14 @@(reduced? result')(assoc loop-state :status :reduced :result @result')- (should-continue? iteration max-iter parts)+ (should-continue? iteration max-iter terminal-tools parts)(assoc loop-state :result result' :iteration (inc iteration)):else(do (log/info "Agent loop complete"{:iterations iteration;; TODO: decide if we want this reason to float up to frontend- :reason (finish-reason iteration max-iter parts)})+ :reason (finish-reason iteration max-iter terminal-tools parts)})(assoc loop-state:status :done:result (rf result' (final-state-part @memory-atom))))))))))diff --git a/src/metabase/metabot/agent/profiles.clj b/src/metabase/metabot/agent/profiles.cljindex 276a345eaec..486c960317d 100644--- a/src/metabase/metabot/agent/profiles.clj+++ b/src/metabase/metabot/agent/profiles.clj@@ -53,21 +53,40 @@- :max-iterations - Maximum agent loop iterations- :temperature - LLM temperature setting- :tools - Vector of tool vars (e.g. #'tools/search-tool)+ - :always-on-skills - Optional vector of skill ids (keywords) whose bodies are inlined into this+ profile's system prompt instead of being loaded on demand via `load_skill`. Always-on is a+ per-profile decision: the same skill can be inlined here and on-demand elsewhere.+ - :terminal-tools - Optional set of tool-name strings whose **successful** call ends the agent+ turn for this profile. Lets a `:required-tool-call?` profile stop as soon as it produces its+ answer (e.g. `:sql` after `edit_sql_query`) instead of being forced to keep calling tools.+ Terminality is per-profile: the same tool is non-terminal in profiles that don't list it.- Tool vars are validated at registration time to ensure they have required metadata."+ Tool vars are validated at registration time to ensure they have required metadata; any+ `:always-on-skills` are validated to refer to registered skills, and any `:terminal-tools` to+ refer to tools the profile actually exposes."[profile :- [:map[:name :keyword][:prompt-template :string][:max-iterations :int][:temperature :float]- [:tools [:vector :any]]]]- (let [tool-vars (:tools profile)]+ [:tools [:vector :any]]+ [:always-on-skills {:optional true} [:vector :keyword]]+ [:terminal-tools {:optional true} [:set :string]]]]+ (let [tool-vars (:tools profile)+ tool-name-seq (map #(:tool-name (meta %)) tool-vars)+ tool-names (set tool-name-seq)](doseq [tool-var tool-vars](validate-tool-var! tool-var))- (when-not (apply distinct? (map #(:tool-name (meta %)) tool-vars))- (let [dups (->> (frequencies (map #(:tool-name (meta %)) tool-vars))+ (when-not (apply distinct? tool-name-seq)+ (let [dups (->> (frequencies tool-name-seq)(filter (fn [[_ cnt]] (< 1 cnt))))]- (throw (ex-info "Duplicate tool names in profile" {:tool-names (map first dups)})))))+ (throw (ex-info "Duplicate tool names in profile" {:tool-names (map first dups)}))))+ (when-let [unknown (seq (remove skills/get-skill (:always-on-skills profile)))]+ (throw (ex-info "Profile references unknown always-on skill ids"+ {:profile (:name profile) :unknown-skill-ids unknown})))+ (when-let [unknown (seq (remove tool-names (:terminal-tools profile)))]+ (throw (ex-info "Profile lists terminal tools it does not expose"+ {:profile (:name profile) :unknown-terminal-tools unknown}))))(swap! *profiles assoc (:name profile) profile))(register-profile!@@ -125,6 +144,20 @@:max-iterations 20:temperature 0.3:required-tool-call? true+ ;; The SQL editor is a focused, tool-heavy flow: this guidance is relevant on essentially every+ ;; turn, so inline it rather than make the model spend iterations loading it. Other profiles that+ ;; share these tools (e.g. :internal) keep them on-demand by not listing them here.+ :always-on-skills [:read-resource+ :create-sql-query+ :edit-sql-query+ :replace-sql-query+ :ask-for-sql-clarification]+ ;; Delivering SQL via one of these tools — or asking the user to clarify — IS the answer here, so a+ ;; successful call ends the turn rather than forcing more tool calls under :required-tool-call?.+ ;; Failed calls don't terminate, so the model can still self-correct. The same tools stay+ ;; non-terminal in other profiles.+ :terminal-tools #{"create_sql_query" "edit_sql_query" "replace_sql_query"+ "ask_for_sql_clarification"}:tools [#'tools/sql-search-tool#'tools/read-resource-tool#'tools/create-sql-query-code-edit-tool@@ -150,6 +183,9 @@:max-iterations 10:temperature 0.3:required-tool-call? true+ ;; Producing a chart draft is the answer; a successful construct ends the turn (schema collection+ ;; is a non-terminal preparatory step). Failed constructs don't terminate, so the model retries.+ :terminal-tools #{"document_construct_model_chart" "document_construct_sql_chart"}:tools [#'tools/list-available-data-sources-tool#'tools/list-available-fields-tool#'tools/get-field-values-tool@@ -218,11 +254,16 @@`load_skill` tool is injected so the agent can pull skill bodies on demand."[profile-id capabilities](when-let [profile (get-profile profile-id)]- (let [base (-> profile- :tools- (filter-by-capabilities capabilities)- filter-by-scope- tool-map)]+ (let [base (-> profile+ :tools+ (filter-by-capabilities capabilities)+ filter-by-scope+ tool-map)+ manifest (skills/build-skill-manifest profile (keys base) capabilities)](cond-> base- (seq (:catalog (skills/build-skill-manifest profile (keys base) capabilities)))+ ;; Register load_skill whenever the profile has ANY skills — on-demand (catalog) or+ ;; always-on. Always-on bodies are inlined, but their presence still implies a+ ;; skill-bearing, possibly dialect-capable profile whose `dialect-preload-parts` emit a+ ;; synthetic `load_skill` call that must resolve to a registered tool.+ (or (seq (:catalog manifest)) (seq (:always-on manifest)))(assoc "load_skill" #'tools/load-skill-tool)))))diff --git a/src/metabase/metabot/agent/prompts.clj b/src/metabase/metabot/agent/prompts.cljindex c1712b88a26..e132466c225 100644--- a/src/metabase/metabot/agent/prompts.clj+++ b/src/metabase/metabot/agent/prompts.clj@@ -181,7 +181,11 @@:first_day_of_week first-day-of-week:sql_dialect sql-dialect:sql_dialect_loaded (some? (skills/dialect-skill sql-dialect))- :skill_catalog catalog+ ;; `not-empty` so an empty catalog is nil (falsy) — Selmer treats+ ;; an empty vector as truthy, which would render the "# Available+ ;; skills … load the skill(s) you need" header with nothing to+ ;; load, nudging the model into pointless `load_skill` calls.+ :skill_catalog (not-empty catalog):skill_always_on (mapv :body always-on):viewing_context viewing-context:recent_views recent-viewsdiff --git a/src/metabase/metabot/skills.clj b/src/metabase/metabot/skills.cljindex 8f521aa8ab2..859efb96825 100644--- a/src/metabase/metabot/skills.clj+++ b/src/metabase/metabot/skills.clj@@ -43,7 +43,6 @@[:profiles {:optional true} [:maybe [:vector :keyword]]][:capabilities {:optional true} [:vector :keyword]][:scope {:optional true} :keyword]- [:always-on {:optional true} :boolean][:priority {:optional true} :int][:dialect {:optional true} :string]])@@ -68,8 +67,7 @@(:profiles skill) (update :profiles #(mapv keyword %))(:capabilities skill) (update :capabilities #(mapv keyword %))(:scope skill) (update :scope keyword)- true (update :priority (fnil int 50))- true (update :always-on boolean)))+ true (update :priority (fnil int 50))))(defn- validate-skill![skill]@@ -244,20 +242,31 @@(defn build-skill-manifest"Build the skill manifest for a profile + active tools, gated by `capabilities` and the user's scope.Returns {:always-on [skill-map…] :catalog [{:id :title :description}…]} sorted by descending priority- then id."+ then id.++ Whether a skill is *always-on* (body inlined in the system prompt) vs *on-demand* (advertised in the+ catalog, loaded via `load_skill`) is decided **per profile**, by the profile's `:always-on-skills`+ set — not by the skill itself — so the same skill can be inlined for one profile and on-demand for+ another (e.g. the SQL skills are always-on for `:sql` but on-demand for `:internal`)."[profile active-tool-names capabilities]- (let [cap-set (capabilities/capability-set capabilities)- visible (->> (skills-for-profile profile active-tool-names)- (filter #(skill-visible? % cap-set))- (sort-by (juxt (comp - :priority) (comp str :id))))]- (record-loadable-skill-ids! visible)- {:always-on (filter :always-on visible)- :catalog (->> visible- (remove :always-on)- (mapv (fn [s]- {:id (name (:id s))- :title (:title s)- :description (:description s)})))}))+ (let [cap-set (capabilities/capability-set capabilities)+ always-on-ids (set (:always-on-skills profile))+ always-on? (comp always-on-ids :id)+ visible (->> (skills-for-profile profile active-tool-names)+ (filter #(skill-visible? % cap-set))+ (sort-by (juxt (comp - :priority) (comp str :id))))+ ;; Always-on skill bodies are already inlined in the system prompt, so they are neither+ ;; advertised in the catalog nor loadable on demand — only the on-demand skills are.+ ;; Recording only the loadable ids also stops `load_skill` from re-fetching an always-on+ ;; skill (a wasted iteration the model would otherwise spend).+ loadable (remove always-on? visible)]+ (record-loadable-skill-ids! loadable)+ {:always-on (filter always-on? visible)+ :catalog (mapv (fn [s]+ {:id (name (:id s))+ :title (:title s)+ :description (:description s)})+ loadable)}));;; Dialect preload@@ -292,11 +301,11 @@Dialect context only arises in SQL-editor sessions, so its presence is itself the gate.Invariant: the emitted pair references the `load_skill` tool, which- [[metabase.metabot.agent.profiles/get-tools-for-profile]] registers whenever the skill catalog is- non-empty.- Any profile that can reach SQL-editor/dialect context also exposes query skills (the cross-cutting- `read-resource` skill alone keeps the catalog non-empty), so `load_skill` is always registered when- this returns a non-empty preload."+ [[metabase.metabot.agent.profiles/get-tools-for-profile]] registers whenever the skill manifest is+ non-empty — i.e. the profile has any catalog or always-on skill.+ Any profile that can reach SQL-editor/dialect context also exposes query skills (e.g. the SQL+ profile inlines the `read-resource` skill via its `:always-on-skills`), so the manifest is non-empty+ and `load_skill` is always registered when this returns a non-empty preload."[engine](or (when-let [skill (dialect-skill engine)](let [skill-id (:id skill)diff --git a/src/metabase/metabot/tools/clarification.clj b/src/metabase/metabot/tools/clarification.cljindex 8ec2416953f..9230aab7604 100644--- a/src/metabase/metabot/tools/clarification.clj+++ b/src/metabase/metabot/tools/clarification.clj@@ -12,8 +12,9 @@"Ask the user for clarification about their SQL query request.This tool is used when the agent needs more information from the user- to properly complete their SQL request. It returns a final response- that stops the agent loop and waits for user input.+ to properly complete their SQL request. Profiles that list+ `ask_for_sql_clarification` in their `:terminal-tools` (e.g. `:sql`) end the+ turn on a successful call and wait for user input.Parameters:- question: The clarification question to ask the user@@ -21,14 +22,12 @@Returns map with:- :structured-output - The question and options for the LLM- - :final-response? - Signals that the agent should stop and wait for user input- :instructions - Instructions for the LLM"[{:keys [question options]}](log/info "Asking for SQL clarification" {:question question:option-count (count options)}){:structured-output {:question question:options (or options [])}- :final-response? true:instructions "The clarification question has been presented to the user. Stop and wait for their response before continuing."})(defn- format-clarification-outputdiff --git a/src/metabase/metabot/tools/document.clj b/src/metabase/metabot/tools/document.cljindex c4f0e8a027a..53e599541b0 100644--- a/src/metabase/metabot/tools/document.clj+++ b/src/metabase/metabot/tools/document.clj@@ -171,8 +171,7 @@:query query:result-type :chart-draft}]{:output "Draft chart payload generated from SQL query."- :structured-output structured- :final-response? true}))))+ :structured-output structured}))))(catch Exception e(log/error e "Error constructing SQL chart draft")(if (:agent-error? (ex-data e))@@ -216,8 +215,7 @@:chart_type chart-type:query_id query-id:query dataset-query- :result-type :chart-draft}- :final-response? true}+ :result-type :chart-draft}};; Preserve tool error messaging from construct_notebook_query path.(or result{:output "Failed to construct model chart draft."})))diff --git a/test/metabase/metabot/agent/core_test.clj b/test/metabase/metabot/agent/core_test.cljindex d97ffeb8584..00a3b35a0fd 100644--- a/test/metabase/metabot/agent/core_test.clj+++ b/test/metabase/metabot/agent/core_test.clj@@ -34,22 +34,48 @@{:type :tool-input :id "t1"}]))))(deftest should-continue-test- (let [max-iter 3]+ (let [max-iter 3+ no-term #{}](testing "continues when iteration < max and has tool calls"- (is (#'agent/should-continue? 0 max-iter [{:type :tool-input}]))- (is (#'agent/should-continue? 1 max-iter [{:type :tool-input}])))+ (is (#'agent/should-continue? 0 max-iter no-term [{:type :tool-input}]))+ (is (#'agent/should-continue? 1 max-iter no-term [{:type :tool-input}])))(testing "continues when text AND tool calls present (LLM thinking aloud)"- (is (#'agent/should-continue? 0 max-iter [{:type :tool-input}- {:type :text}]))- (is (#'agent/should-continue? 0 max-iter [{:type :text}- {:type :tool-input}])))+ (is (#'agent/should-continue? 0 max-iter no-term [{:type :tool-input}+ {:type :text}]))+ (is (#'agent/should-continue? 0 max-iter no-term [{:type :text}+ {:type :tool-input}])))(testing "stops at max iterations (1-based: iteration >= max means done)"- (is (not (#'agent/should-continue? 3 max-iter [{:type :tool-input}])))- (is (not (#'agent/should-continue? 4 max-iter [{:type :tool-input}]))))+ (is (not (#'agent/should-continue? 3 max-iter no-term [{:type :tool-input}])))+ (is (not (#'agent/should-continue? 4 max-iter no-term [{:type :tool-input}]))))(testing "stops when no tool calls (text-only is final answer)"- (is (not (#'agent/should-continue? 0 max-iter [{:type :text}])))- (is (not (#'agent/should-continue? 0 max-iter [{:type :usage}])))- (is (not (#'agent/should-continue? 0 max-iter []))))))+ (is (not (#'agent/should-continue? 0 max-iter no-term [{:type :text}])))+ (is (not (#'agent/should-continue? 0 max-iter no-term [{:type :usage}])))+ (is (not (#'agent/should-continue? 0 max-iter no-term []))))))++(deftest terminal-tool-call-test+ (let [terminal #{"edit_sql_query" "create_sql_query" "replace_sql_query"}+ success [{:type :tool-input :id "a" :function "edit_sql_query"}+ {:type :tool-output :id "a" :result {:output "ok"+ :structured-output {:result-type :query+ :query-id "q1"}}}]+ failure [{:type :tool-input :id "b" :function "edit_sql_query"}+ {:type :tool-output :id "b" :result {:output "validation error"+ :instructions "fix it"}}]+ read [{:type :tool-input :id "c" :function "read_resource"}+ {:type :tool-output :id "c" :result {:output "<fields/>"}}]]+ (testing "a successful terminal-tool call ends the turn"+ (is (#'agent/terminal-tool-call? terminal success))+ (is (not (#'agent/should-continue? 0 20 terminal success))+ "should-continue? is false right after a successful edit (the 10402 fix)"))+ (testing "a FAILED terminal-tool call does not end the turn (model can self-correct)"+ (is (not (#'agent/terminal-tool-call? terminal failure)))+ (is (#'agent/should-continue? 0 20 terminal failure)))+ (testing "a non-terminal tool (read_resource) does not end the turn"+ (is (not (#'agent/terminal-tool-call? terminal read))))+ (testing "terminality is per-profile: an empty terminal set never ends the turn"+ (is (not (#'agent/terminal-tool-call? #{} success))))+ (testing "finish-reason reports :terminal-tool"+ (is (= :terminal-tool (#'agent/finish-reason 0 20 terminal success))))))(deftest run-agent-loop-with-mock-test(mt/as-admindiff --git a/test/metabase/metabot/agent/profiles_test.clj b/test/metabase/metabot/agent/profiles_test.cljindex ca5c246317e..9bc90cff472 100644--- a/test/metabase/metabot/agent/profiles_test.clj+++ b/test/metabase/metabot/agent/profiles_test.clj@@ -4,6 +4,7 @@[metabase.api-scope.core :as api-scope][metabase.metabot.agent.profiles :as profiles][metabase.metabot.scope :as scope]+ [metabase.metabot.tools :as tools][metabase.metabot.tools.transforms :as tools.transforms][metabase.premium-features.core :as premium-features][metabase.test :as mt]))@@ -215,3 +216,29 @@(is (= #{}(set (#'profiles/filter-by-capabilities transform-tools["permission:write_transforms"]))))))))++(deftest terminal-tools-test+ (testing "the :sql profile marks its SQL write tools AND clarification terminal"+ (is (= #{"create_sql_query" "edit_sql_query" "replace_sql_query" "ask_for_sql_clarification"}+ (:terminal-tools (profiles/get-profile :sql)))))+ (testing "the document profile ends the turn on a constructed chart, not on schema collection"+ (is (= #{"document_construct_model_chart" "document_construct_sql_chart"}+ (:terminal-tools (profiles/get-profile :document-generate-content)))))+ (testing "terminality is per-profile — profiles that share these tools don't inherit it"+ (is (nil? (:terminal-tools (profiles/get-profile :internal))))+ (is (nil? (:terminal-tools (profiles/get-profile :nlq))))))++(deftest register-profile-validation-test+ (let [base {:name :scratch+ :prompt-template "internal.selmer"+ :max-iterations 10+ :temperature 0.3+ :tools [#'tools/read-resource-tool]}]+ (testing "rejects :always-on-skills that don't resolve to a registered skill"+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"unknown always-on skill"+ (#'profiles/register-profile!+ (assoc base :always-on-skills [:no-such-skill])))))+ (testing "rejects :terminal-tools the profile does not expose"+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"terminal tools it does not expose"+ (#'profiles/register-profile!+ (assoc base :terminal-tools #{"nonexistent_tool"})))))))diff --git a/test/metabase/metabot/skills_test.clj b/test/metabase/metabot/skills_test.cljindex def21a2ff5a..29e1af3935a 100644--- a/test/metabase/metabot/skills_test.clj+++ b/test/metabase/metabot/skills_test.clj@@ -2,6 +2,7 @@(:require[clojure.string :as str][clojure.test :refer [deftest is testing]]+ [metabase.metabot.agent.prompts :as prompts][metabase.metabot.scope :as scope][metabase.metabot.skills :as skills][metabase.test :as mt]))@@ -85,30 +86,90 @@(is (empty? (filter :dialect (skills/skills-for-profile profile ["construct_notebook_query"])))))))(deftest ^:parallel build-skill-manifest-test- (let [profile {:name :internal}+ ;; :internal declares no :always-on-skills, so every relevant skill is on-demand (catalog).+ (let [profile {:name :internal}manifest (skills/build-skill-manifestprofile["construct_notebook_query" "read_resource"][])]- (testing "catalog lists available skills with string ids"+ (testing "catalog lists available on-demand skills with string ids"(let [ids (map :id (:catalog manifest))](is (every? string? ids))(is (some #{"construct-notebook-query-core"} ids))- (is (some #{"read-resource"} ids))))+ (is (some #{"read-resource"} ids)+ "without :always-on-skills, read-resource is on-demand for this profile")))(testing "catalog entries are display-only (id/title/description), no body"(is (every? #(= #{:id :title :description} (set (keys %))) (:catalog manifest))))(testing "catalog is sorted by descending priority then id";; core has priority 60, the rest 50, operators 40 -> core first, operators last(is (= "construct-notebook-query-core" (:id (first (:catalog manifest))))))- (testing "no always-on skills in this migration"+ (testing "no always-on skills when the profile declares none"(is (empty? (:always-on manifest))))))+(deftest ^:parallel build-skill-manifest-always-on-is-per-profile-test+ (testing "a skill listed in the profile's :always-on-skills is inlined (body) and removed from+ the catalog, while the same skill is on-demand for a profile that doesn't list it"+ (let [tools ["read_resource"]+ inlined (skills/build-skill-manifest+ {:name :sql :always-on-skills [:read-resource]} tools [])+ on-demand (skills/build-skill-manifest+ {:name :internal} tools [])]+ (testing "profile that opts in -> always-on with body, absent from catalog"+ (is (= ["read-resource"] (map (comp name :id) (:always-on inlined))))+ (is (every? :body (:always-on inlined)))+ (is (not (some #{"read-resource"} (map :id (:catalog inlined))))))+ (testing "profile that opts out -> on-demand in catalog, not always-on"+ (is (empty? (:always-on on-demand)))+ (is (some #{"read-resource"} (map :id (:catalog on-demand))))))))++(deftest ^:parallel always-on-skill-body-is-in-system-prompt-test+ ;; A "marker" line lifted verbatim from the read-resource skill body. If the skill is inlined,+ ;; this text appears in the rendered system prompt; if it is on-demand, only its one-line catalog+ ;; description appears (which does not contain this phrase).+ (let [marker "Drill, don't re-search"+ tools {"read_resource" nil}+ render (fn [profile]+ (binding [scope/*current-user-metabot-permissions*+ {:permission/metabot-sql-generation :yes+ :permission/metabot-nlq :no+ :permission/metabot-other-tools :no}]+ (prompts/build-system-message-content+ (assoc profile :prompt-template "sql-querying-only.selmer")+ {:current_time "2026-06-19T12:00:00Z"}+ tools+ [])))]+ (testing "read-resource body IS present when the profile marks it always-on"+ (let [rendered (render {:name :sql :always-on-skills [:read-resource]})]+ (is (str/includes? rendered marker)+ "always-on skill body must be inlined into the system prompt")+ (is (str/includes? rendered (:body (skills/get-skill :read-resource)))+ "the full skill body is inlined, not just a fragment")+ (is (not (str/includes? rendered "# Available skills"))+ "an empty catalog must NOT render the load_skill header (would invite pointless loads)")))+ (testing "read-resource body is ABSENT when the profile leaves it on-demand"+ (let [rendered (render {:name :sql})]+ (is (not (str/includes? rendered marker))+ "an on-demand skill's body must not leak into the prompt prefix")+ (is (str/includes? rendered "# Available skills")+ "a non-empty catalog renders the load_skill header")+ (is (str/includes? rendered "read-resource")+ "the on-demand skill still appears as a one-line catalog entry")))))+(deftest ^:parallel build-skill-manifest-records-loadable-skill-ids-test(testing "building the manifest records the request's loadable skill ids"(binding [scope/*current-loadable-skill-ids* (atom #{})](skills/build-skill-manifest {:name :internal} ["construct_notebook_query"] [])(is (contains? @scope/*current-loadable-skill-ids* :construct-notebook-query-core))- (is (not (contains? @scope/*current-loadable-skill-ids* :read-resource))))))+ (is (not (contains? @scope/*current-loadable-skill-ids* :read-resource)))))+ (testing "a profile's always-on skills are NOT loadable — their bodies are already inlined, so+ load_skill must not re-fetch them (a wasted iteration)"+ (binding [scope/*current-loadable-skill-ids* (atom #{})]+ (skills/build-skill-manifest+ {:name :sql :always-on-skills [:read-resource]} ["read_resource"] [])+ (is (not (contains? @scope/*current-loadable-skill-ids* :read-resource))+ "an always-on skill must be excluded from the loadable set")+ (is (not (skills/skill-loadable? (skills/get-skill :read-resource)))+ "skill-loadable? rejects an always-on skill once a manifest is tracked"))))(deftest ^:parallel dialect-preload-parts-test(testing "resolves a synthetic load_skill pair when the dialect is known"diff --git a/test/metabase/metabot/tools/clarification_test.clj b/test/metabase/metabot/tools/clarification_test.cljindex ba4bde17485..13438c75def 100644--- a/test/metabase/metabot/tools/clarification_test.clj+++ b/test/metabase/metabot/tools/clarification_test.clj@@ -5,10 +5,10 @@[metabase.metabot.tools.clarification :as ask-clarification]))(deftest ask-for-sql-clarification-test- (testing "returns final-response? flag"+ (testing "returns structured-output (the success signal that lets a profile end the turn)"(let [result (ask-clarification/ask-for-sql-clarification {:question "What table should I use?"})](is (map? result))- (is (true? (:final-response? result)))))+ (is (some? (:structured-output result)))))(testing "returns question in structured output"(let [result (ask-clarification/ask-for-sql-clarification {:question "What columns do you need?"})](is (= "What columns do you need?" (get-in result [:structured-output :question])))))@@ -26,10 +26,13 @@(is (string? (:instructions result)))(is (str/includes? (:instructions result) "wait")))))-(deftest final-response-behavior-test- (testing "final-response? signals agent should stop"- ;; This is a contract test - the agent loop checks for :final-response? true- ;; to determine if it should stop processing+(deftest terminal-via-structured-output-test+ (testing "the tool produces :structured-output so a profile with it in :terminal-tools stops"+ ;; Contract: termination is profile-driven (see metabase.metabot.agent.profiles/:sql+ ;; :terminal-tools). The agent loop ends the turn on a *successful* terminal-tool call, and a+ ;; successful clarification is signalled by the presence of :structured-output.(let [result (ask-clarification/ask-for-sql-clarification {:question "Test?"})]- (is (true? (:final-response? result))- "ask_for_sql_clarification must return :final-response? true to stop the agent loop"))))+ (is (some? (:structured-output result))+ "ask_for_sql_clarification must return :structured-output so the agent loop can stop")+ (is (not (contains? result :final-response?))+ "the tool-level :final-response? flag has been removed in favour of :terminal-tools"))))diff --git a/test/metabase/metabot/tools/document_test.clj b/test/metabase/metabot/tools/document_test.cljindex 073d0474631..9f7ff96aa2e 100644--- a/test/metabase/metabot/tools/document_test.clj+++ b/test/metabase/metabot/tools/document_test.clj@@ -57,7 +57,8 @@:sql "SELECT * FROM test":viz_settings {:chart_type "bar"}})structured (:structured-output result)]- (is (true? (:final-response? result)))+ (is (some? structured)+ "structured-output present -> a successful, terminal-eligible call")(is (= "document_construct_sql_chart" (:tool structured)))(is (= "Test Name" (:name structured)))(is (= "Test Desc" (:description structured)))@@ -85,8 +86,8 @@:approach "Test Approach":sql "SELECT FROM test":viz_settings {:chart_type "bar"}})]- (is (nil? (:final-response? result)))- (is (nil? (:structured-output result)))+ (is (nil? (:structured-output result))+ "no structured-output -> a failure, which must NOT terminate the turn")(is (re-find #"SQL chart draft generation failed" (:output result)))(is (re-find #"syntax error near FROM" (:output result)))))))@@ -111,8 +112,8 @@:approach "Test Approach":sql "SELECT * FROM missing_table":viz_settings {:chart_type "bar"}})]- (is (nil? (:final-response? result)))- (is (nil? (:structured-output result)))+ (is (nil? (:structured-output result))+ "no structured-output -> a failure, which must NOT terminate the turn")(is (re-find #"could not be processed by Metabase" (:output result)))(is (re-find #"missing_table" (:output result)))))))@@ -131,7 +132,8 @@:query "":viz_settings {:chart_type "bar"}})structured (:structured-output result)]- (is (true? (:final-response? result)))+ (is (some? structured)+ "structured-output present -> a successful, terminal-eligible call")(is (= "document_construct_model_chart" (:tool structured)))(is (= "Test Name" (:name structured)))(is (= "Test Desc" (:description structured)))