The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Let Metabot build a query without showing a chart (#84477)
Chris Truter513ccc60d82f
diff --git a/resources/metabot/prompts/shared/prompt_snippets/results_notebook_values.selmer b/resources/metabot/prompts/shared/prompt_snippets/results_notebook_values.selmernew file mode 100644index 00000000000..4e14cb0b861--- /dev/null+++ b/resources/metabot/prompts/shared/prompt_snippets/results_notebook_values.selmer@@ -0,0 +1 @@+When you build a notebook query only to read its values, pass `show_chart: false` to `construct_notebook_query`, so the user is not shown a chart they did not ask for. The user sees nothing of such a query, so give them the values in your answer. Leave the argument out when the user asked to see the data or a chart.diff --git a/resources/metabot/prompts/shared/results.selmer b/resources/metabot/prompts/shared/results.selmerindex 7732907c9c7..9fb2a58aeb3 100644--- a/resources/metabot/prompts/shared/results.selmer+++ b/resources/metabot/prompts/shared/results.selmer@@ -1,3 +1,4 @@{% if has_query_execution %}{% include "metabot/prompts/shared/prompt_snippets/results_visible.selmer" %}-{% if has_notebook_query %}{% include "metabot/prompts/shared/prompt_snippets/results_saved_question_notebook.selmer" %}{% else %}{% include "metabot/prompts/shared/prompt_snippets/results_saved_question_sql.selmer" %}{% endif %}+{% if has_notebook_query %}{% include "metabot/prompts/shared/prompt_snippets/results_saved_question_notebook.selmer" %}+{% include "metabot/prompts/shared/prompt_snippets/results_notebook_values.selmer" %}{% else %}{% include "metabot/prompts/shared/prompt_snippets/results_saved_question_sql.selmer" %}{% endif %}{% if has_sql_execution %}{% if has_notebook_query %}{% include "metabot/prompts/shared/prompt_snippets/results_sql.selmer" %}{% else %}{% include "metabot/prompts/shared/prompt_snippets/results_sql_only.selmer" %}{% endif %}{% else %}{% include "metabot/prompts/shared/prompt_snippets/results_notebook_only.selmer" %}{% endif %}{% else %}{% include "metabot/prompts/shared/prompt_snippets/results_hidden.selmer" %}{% endif %}diff --git a/src/metabase/metabot/agent/core.clj b/src/metabase/metabot/agent/core.cljindex f28b215ebe5..3295dd5609f 100644--- a/src/metabase/metabot/agent/core.clj+++ b/src/metabase/metabot/agent/core.clj@@ -523,7 +523,11 @@:request-canceled-chan request-canceled-chan)(memory/add-client-ids (client-content-ids context)))memory-atom (doto (or external-memory-atom (atom nil)) (reset! memory))- tools (update-vals (tools/wrap-tools-with-state base-tools memory-atom metabot profile-id)+ ;; The show_chart argument only does anything alongside run_query, so a turn without run_query declares+ ;; construct_notebook_query as it was before the argument existed.+ tools (update-vals (cond-> (tools/wrap-tools-with-state base-tools memory-atom metabot profile-id)+ (not (contains? base-tools "run_query"))+ (m/update-existing "construct_notebook_query" tools/without-show-chart))#(assoc % :declaration (delay (self.schema/tool-function %))))](log/info "Starting agent" {:profile profile-id:tools (count tools)diff --git a/src/metabase/metabot/tools.clj b/src/metabase/metabot/tools.cljindex 579531b3077..7880b3f7d0d 100644--- a/src/metabase/metabot/tools.clj+++ b/src/metabase/metabot/tools.clj@@ -48,7 +48,8 @@[tools.entity-retrievalretrieve-library-entities-tool][tools.construct- construct-notebook-query-tool]+ construct-notebook-query-tool+ without-show-chart][tools.run-queryrun-query-toolrun-sql-query-tool]diff --git a/src/metabase/metabot/tools/construct.clj b/src/metabase/metabot/tools/construct.cljindex d7aaf184a31..b699db55101 100644--- a/src/metabase/metabot/tools/construct.clj+++ b/src/metabase/metabot/tools/construct.clj@@ -18,6 +18,7 @@[metabase.metabot.tmpl :as te][metabase.metabot.tools.charts.create :as create-chart-tools][metabase.metabot.tools.recovery-hints :as recovery-hints]+ [metabase.metabot.tools.shared :as shared][metabase.metabot.tools.shared.content-store :as shared.content-store][metabase.metabot.tools.shared.instructions :as instructions][metabase.metabot.tools.shared.llm-shape :as llm-shape]@@ -121,9 +122,23 @@[:reasoning {:optional true} :string][:query LLMQueryArgument][:visualization {:optional true} construct-visualization-schema]+ [:show_chart {:optional true+ :description (str "Pass false for a query you build only to read its values with run_query: "+ "the user is then shown no chart. Defaults to true.")}+ [:maybe :boolean]][:title :string][:description :string]])+(defn without-show-chart+ "Returns the construct_notebook_query tool definition `tool` with `show_chart` left out of its `:schema`, so the+ model is not told about the argument."+ [tool]+ (update tool :schema (fn [[arrow [cat args-schema & more] output]]+ [arrow+ (into [cat (into [] (remove #(and (vector? %) (= :show_chart (first %)))) args-schema)]+ more)+ output])))+;;; ---------------------------------------- Source resolution ----------------------------------------(defn- portable-table-fk@@ -839,6 +854,12 @@:queries [(structured->query-data structured)]:visualization_settings {:chart_type (if chart-type (name chart-type) "table")}}))+(defn- tool-output+ "The text the model sees: `result-xml` followed by `instruction-text`."+ [result-xml instruction-text]+ (str "<result>\n" result-xml "\n</result>\n"+ "<instructions>\n" instruction-text "\n</instructions>"))+;;; ---------------------------------------- Main tool ----------------------------------------(mu/defn ^{:tool-name "construct_notebook_query"@@ -851,7 +872,7 @@one- or two-sentence `description` of what the chart shows (the metric, the grouping, andany notable filter); it is used as the saved question's description. See`resources/metabot/prompts/tools/construct_notebook_query.md` for the prompt contract."- [{:keys [_reasoning query visualization title description]} :- construct-notebook-query-args-schema]+ [{:keys [_reasoning query visualization show_chart title description]} :- construct-notebook-query-args-schema](try(let [normalized-visualization (some-> visualization (update-keys (comp keyword u/->kebab-case-en name)))chart-type (or (chart-type->keyword (:chart-type normalized-visualization))@@ -859,8 +880,30 @@query-result (execute-representations-queryquery{:recovery-hint recovery-hints/recovery-hint})- structured (or (:structured-output query-result) (:structured_output query-result))]- (if (and structured (:query-id structured) (:query structured))+ structured (or (:structured-output query-result) (:structured_output query-result))+ ;; A query built only to be read needs run_query, so without it the chart is the one way the user sees+ ;; anything and is always shown.+ show-chart? (not (and (false? show_chart) (shared/tool-offered? "run_query")))]+ (cond+ (not (and structured (:query-id structured) (:query structured)))+ ;; query-result may already have :output (error) or only :structured-output+ (if structured+ (let [query-xml (llm-shape/query->xml (structured->query-data structured))+ instruction-text (instructions/query-created-instructions-for (:query-id structured) false)]+ (assoc query-result :output (tool-output query-xml instruction-text)))+ query-result)++ (not show-chart?)+ (let [instruction-text (te/lines+ "Your query has been created. The user has not been shown it."+ ""+ (str "To read its values, call `run_query` with `query_id` `"+ (:query-id structured) "`."))]+ {:output (tool-output (llm-shape/query->xml (structured->query-data structured)) instruction-text)+ :structured-output (assoc structured :result-type :query)+ :instructions instruction-text})++ :else(let [chart-result (create-chart-tools/create-chart{:query-id (:query-id structured):chart-type chart-type@@ -880,8 +923,7 @@(str "- Always provide a direct link using: `" link "` where Chart is a meaningful link text")"- If creating multiple charts, present all chart links"))chart-xml (structured->chart-xml structured (:chart-id chart-result) chart-type)]- {:output (str "<result>\n" chart-xml "\n</result>\n"- "<instructions>\n" instruction-text "\n</instructions>")+ {:output (tool-output chart-xml instruction-text):data-parts [(streaming/viz-part{:entity-id (:chart-id chart-result):query-id (:query-id structured)@@ -890,14 +932,6 @@:title title:description description})]:structured-output full-structured- :instructions instruction-text})- ;; query-result may already have :output (error) or only :structured-output- (if-let [s (or (:structured-output query-result) (:structured_output query-result))]- (let [query-xml (llm-shape/query->xml (structured->query-data s))- instruction-text (instructions/query-created-instructions-for (:query-id s) false)]- (assoc query-result- :output (str "<result>\n" query-xml "\n</result>\n"- "<instructions>\n" instruction-text "\n</instructions>")))- query-result)))+ :instructions instruction-text})))(catch Exception e(tools.u/handle-agent-or-api-error e))))diff --git a/test/metabase/metabot/agent/core_test.clj b/test/metabase/metabot/agent/core_test.cljindex 9ee68561d18..84a5d8f1b78 100644--- a/test/metabase/metabot/agent/core_test.clj+++ b/test/metabase/metabot/agent/core_test.clj@@ -158,9 +158,9 @@(is (contains? tools "replace_sql_query")))))))(defn- run-query-offered-at-turn-start- "Whether the turn starts with `run_query` among its tools, and whether its system message says results can be read.- Reads them where the decision is made rather than running a turn. The bindings stand in for the ones the agent- loop makes."+ "Whether the turn starts with `run_query` among its tools, whether its system message says results can be read, and+ whether it declares `show_chart` on construct_notebook_query. Reads them where the decision is made rather than+ running a turn. The bindings stand in for the ones the agent loop makes."[conversation-id capabilities](let [{:keys [tools system-msg]}(binding [scope/*current-user-metabot-permissions* scope/all-yes-permissions@@ -169,8 +169,13 @@:profile-id :internal:conversation-id conversation-id:context {:capabilities capabilities}}))]- {:tool? (contains? tools "run_query")- :prompt? (str/includes? (pr-str system-msg) "You can see results only")}))+ {:tool? (contains? tools "run_query")+ :prompt? (str/includes? (pr-str system-msg) "You can see results only")+ :show-chart? (-> (get tools "construct_notebook_query")+ :declaration+ deref+ (get-in [:parameters :properties])+ (contains? :show_chart))}))(deftest client-claimed-feature-capability-cannot-override-settings-test;; In a conversation the user owns, so the setting decides and not the rule for shared conversations.@@ -191,15 +196,15 @@:model/MetabotConversation {someone :id} {:user_id (mt/user->id :crowberto)}](mt/with-temporary-setting-values [metabot-query-execution-enabled? true](mt/with-current-user rasta- (testing "the user's own conversation offers run_query and says results can be read"- (is (= {:tool? true, :prompt? true} (offered own))))- (testing "a conversation other people can read offers neither"+ (testing "the user's own conversation offers run_query, says results can be read, and declares show_chart"+ (is (= {:tool? true, :prompt? true, :show-chart? true} (offered own))))+ (testing "a conversation other people can read does none of them"(testing "a Slack thread"- (is (= {:tool? false, :prompt? false} (offered thread))))+ (is (= {:tool? false, :prompt? false, :show-chart? false} (offered thread))))(testing "a conversation someone else started"- (is (= {:tool? false, :prompt? false} (offered someone)))))- (testing "no conversation offers neither"- (is (= {:tool? false, :prompt? false} (offered nil)))))))))+ (is (= {:tool? false, :prompt? false, :show-chart? false} (offered someone)))))+ (testing "no conversation does none of them"+ (is (= {:tool? false, :prompt? false, :show-chart? false} (offered nil)))))))))(deftest document-sql-chart-tool-requires-native-permission-test(mt/with-no-data-perms-for-all-users!diff --git a/test/metabase/metabot/agent/prompt_gating_test.clj b/test/metabase/metabot/agent/prompt_gating_test.cljindex a54c889bc6c..ee5a19522be 100644--- a/test/metabase/metabot/agent/prompt_gating_test.clj+++ b/test/metabase/metabot/agent/prompt_gating_test.clj@@ -323,6 +323,7 @@(let [rendered (render-internal-template all-yes-perms ["construct_notebook_query" "run_query"])](is (re-find #"# You can see results only by running a query" rendered))(is (re-find #"To read a saved question or model.*`source-card`" rendered))+ (is (re-find #"pass `show_chart: false` to `construct_notebook_query`" rendered))(is (not (re-find #"you cannot see query results" rendered))))))(deftest prompt-gates-sql-execution-guidance-testdiff --git a/test/metabase/metabot/tools_test.clj b/test/metabase/metabot/tools_test.cljindex 0c8f757c538..5f4fbaeadcd 100644--- a/test/metabase/metabot/tools_test.clj+++ b/test/metabase/metabot/tools_test.clj@@ -189,6 +189,40 @@(is (= "Total order count."(get-in result [:data-parts 0 :data :description]))))))))+(deftest construct-notebook-query-tool-show-chart-test+ (let [construct! (fn [tool-names show-chart-arg]+ (let [chart-created? (atom false)]+ (mt/with-dynamic-fn-redefs [construct/execute-representations-query+ (fn [_ _]+ {:structured-output {:query-id "q-1"+ :query {:database 1}+ :result-columns []}})+ create-chart-tools/create-chart+ (fn [_]+ (reset! chart-created? true)+ {:chart-id "c-1", :chart-type :table})]+ (let [result (binding [shared/*profile-id* :nlq+ shared/*memory-atom* (atom {:tool-names tool-names})]+ (agent-tools/construct-notebook-query-tool+ (merge {:query {:lib/type "mbql/query", :stages []}+ :title "Seat check"+ :description "Total order count."}+ show-chart-arg)))]+ {:chart? @chart-created?+ :shown? (boolean (seq (:data-parts result)))+ :query-id (get-in result [:structured-output :query-id])+ :run-it? (boolean (re-find #"call `run_query` with `query_id` `q-1`"+ (:output result)))}))))]+ (testing "with run_query offered, the query is built and the user is shown nothing"+ (is (= {:chart? false, :shown? false, :query-id "q-1", :run-it? true}+ (construct! #{"construct_notebook_query" "run_query"} {:show_chart false}))))+ (testing "with run_query offered and the argument left out, the chart is shown"+ (is (= {:chart? true, :shown? true, :query-id "q-1", :run-it? false}+ (construct! #{"construct_notebook_query" "run_query"} {}))))+ (testing "without run_query the chart is shown whatever the argument says"+ (is (= {:chart? true, :shown? true, :query-id "q-1", :run-it? false}+ (construct! #{"construct_notebook_query"} {:show_chart false}))))))+(defn- construct-tool-output-for-thrown"Run `construct_notebook_query` with `execute-representations-query` throwing `e`.Returns the `:output` the LLM would see when the tool handles `e`; otherwise `e` propagates."