Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Let Metabot build a query without showing a chart (#84477)

Chris Truter513ccc60d82f
diff --git a/resources/metabot/prompts/shared/prompt_snippets/results_notebook_values.selmer b/resources/metabot/prompts/shared/prompt_snippets/results_notebook_values.selmer
new file mode 100644
index 00000000000..4e14cb0b861
--- /dev/null
+++ b/resources/metabot/prompts/shared/prompt_snippets/results_notebook_values.selmer
@@ -0,0 +1 @@
+When you build a notebook query only to read its values, pass `show_chart: false` to `construct_notebook_query`, so the user is not shown a chart they did not ask for. The user sees nothing of such a query, so give them the values in your answer. Leave the argument out when the user asked to see the data or a chart.
diff --git a/resources/metabot/prompts/shared/results.selmer b/resources/metabot/prompts/shared/results.selmer
index 7732907c9c7..9fb2a58aeb3 100644
--- a/resources/metabot/prompts/shared/results.selmer
+++ b/resources/metabot/prompts/shared/results.selmer
@@ -1,3 +1,4 @@
{% if has_query_execution %}{% include "metabot/prompts/shared/prompt_snippets/results_visible.selmer" %}
-{% if has_notebook_query %}{% include "metabot/prompts/shared/prompt_snippets/results_saved_question_notebook.selmer" %}{% else %}{% include "metabot/prompts/shared/prompt_snippets/results_saved_question_sql.selmer" %}{% endif %}
+{% if has_notebook_query %}{% include "metabot/prompts/shared/prompt_snippets/results_saved_question_notebook.selmer" %}
+{% include "metabot/prompts/shared/prompt_snippets/results_notebook_values.selmer" %}{% else %}{% include "metabot/prompts/shared/prompt_snippets/results_saved_question_sql.selmer" %}{% endif %}
{% if has_sql_execution %}{% if has_notebook_query %}{% include "metabot/prompts/shared/prompt_snippets/results_sql.selmer" %}{% else %}{% include "metabot/prompts/shared/prompt_snippets/results_sql_only.selmer" %}{% endif %}{% else %}{% include "metabot/prompts/shared/prompt_snippets/results_notebook_only.selmer" %}{% endif %}{% else %}{% include "metabot/prompts/shared/prompt_snippets/results_hidden.selmer" %}{% endif %}
diff --git a/src/metabase/metabot/agent/core.clj b/src/metabase/metabot/agent/core.clj
index f28b215ebe5..3295dd5609f 100644
--- a/src/metabase/metabot/agent/core.clj
+++ b/src/metabase/metabot/agent/core.clj
@@ -523,7 +523,11 @@
:request-canceled-chan request-canceled-chan)
(memory/add-client-ids (client-content-ids context)))
memory-atom (doto (or external-memory-atom (atom nil)) (reset! memory))
- tools (update-vals (tools/wrap-tools-with-state base-tools memory-atom metabot profile-id)
+ ;; The show_chart argument only does anything alongside run_query, so a turn without run_query declares
+ ;; construct_notebook_query as it was before the argument existed.
+ tools (update-vals (cond-> (tools/wrap-tools-with-state base-tools memory-atom metabot profile-id)
+ (not (contains? base-tools "run_query"))
+ (m/update-existing "construct_notebook_query" tools/without-show-chart))
#(assoc % :declaration (delay (self.schema/tool-function %))))]
(log/info "Starting agent" {:profile profile-id
:tools (count tools)
diff --git a/src/metabase/metabot/tools.clj b/src/metabase/metabot/tools.clj
index 579531b3077..7880b3f7d0d 100644
--- a/src/metabase/metabot/tools.clj
+++ b/src/metabase/metabot/tools.clj
@@ -48,7 +48,8 @@
[tools.entity-retrieval
retrieve-library-entities-tool]
[tools.construct
- construct-notebook-query-tool]
+ construct-notebook-query-tool
+ without-show-chart]
[tools.run-query
run-query-tool
run-sql-query-tool]
diff --git a/src/metabase/metabot/tools/construct.clj b/src/metabase/metabot/tools/construct.clj
index d7aaf184a31..b699db55101 100644
--- a/src/metabase/metabot/tools/construct.clj
+++ b/src/metabase/metabot/tools/construct.clj
@@ -18,6 +18,7 @@
[metabase.metabot.tmpl :as te]
[metabase.metabot.tools.charts.create :as create-chart-tools]
[metabase.metabot.tools.recovery-hints :as recovery-hints]
+ [metabase.metabot.tools.shared :as shared]
[metabase.metabot.tools.shared.content-store :as shared.content-store]
[metabase.metabot.tools.shared.instructions :as instructions]
[metabase.metabot.tools.shared.llm-shape :as llm-shape]
@@ -121,9 +122,23 @@
[:reasoning {:optional true} :string]
[:query LLMQueryArgument]
[:visualization {:optional true} construct-visualization-schema]
+ [:show_chart {:optional true
+ :description (str "Pass false for a query you build only to read its values with run_query: "
+ "the user is then shown no chart. Defaults to true.")}
+ [:maybe :boolean]]
[:title :string]
[:description :string]])
+(defn without-show-chart
+ "Returns the construct_notebook_query tool definition `tool` with `show_chart` left out of its `:schema`, so the
+ model is not told about the argument."
+ [tool]
+ (update tool :schema (fn [[arrow [cat args-schema & more] output]]
+ [arrow
+ (into [cat (into [] (remove #(and (vector? %) (= :show_chart (first %)))) args-schema)]
+ more)
+ output])))
+
;;; ---------------------------------------- Source resolution ----------------------------------------
(defn- portable-table-fk
@@ -839,6 +854,12 @@
:queries [(structured->query-data structured)]
:visualization_settings {:chart_type (if chart-type (name chart-type) "table")}}))
+(defn- tool-output
+ "The text the model sees: `result-xml` followed by `instruction-text`."
+ [result-xml instruction-text]
+ (str "<result>\n" result-xml "\n</result>\n"
+ "<instructions>\n" instruction-text "\n</instructions>"))
+
;;; ---------------------------------------- Main tool ----------------------------------------
(mu/defn ^{:tool-name "construct_notebook_query"
@@ -851,7 +872,7 @@
one- or two-sentence `description` of what the chart shows (the metric, the grouping, and
any notable filter); it is used as the saved question's description. See
`resources/metabot/prompts/tools/construct_notebook_query.md` for the prompt contract."
- [{:keys [_reasoning query visualization title description]} :- construct-notebook-query-args-schema]
+ [{:keys [_reasoning query visualization show_chart title description]} :- construct-notebook-query-args-schema]
(try
(let [normalized-visualization (some-> visualization (update-keys (comp keyword u/->kebab-case-en name)))
chart-type (or (chart-type->keyword (:chart-type normalized-visualization))
@@ -859,8 +880,30 @@
query-result (execute-representations-query
query
{:recovery-hint recovery-hints/recovery-hint})
- structured (or (:structured-output query-result) (:structured_output query-result))]
- (if (and structured (:query-id structured) (:query structured))
+ structured (or (:structured-output query-result) (:structured_output query-result))
+ ;; A query built only to be read needs run_query, so without it the chart is the one way the user sees
+ ;; anything and is always shown.
+ show-chart? (not (and (false? show_chart) (shared/tool-offered? "run_query")))]
+ (cond
+ (not (and structured (:query-id structured) (:query structured)))
+ ;; query-result may already have :output (error) or only :structured-output
+ (if structured
+ (let [query-xml (llm-shape/query->xml (structured->query-data structured))
+ instruction-text (instructions/query-created-instructions-for (:query-id structured) false)]
+ (assoc query-result :output (tool-output query-xml instruction-text)))
+ query-result)
+
+ (not show-chart?)
+ (let [instruction-text (te/lines
+ "Your query has been created. The user has not been shown it."
+ ""
+ (str "To read its values, call `run_query` with `query_id` `"
+ (:query-id structured) "`."))]
+ {:output (tool-output (llm-shape/query->xml (structured->query-data structured)) instruction-text)
+ :structured-output (assoc structured :result-type :query)
+ :instructions instruction-text})
+
+ :else
(let [chart-result (create-chart-tools/create-chart
{:query-id (:query-id structured)
:chart-type chart-type
@@ -880,8 +923,7 @@
(str "- Always provide a direct link using: `" link "` where Chart is a meaningful link text")
"- If creating multiple charts, present all chart links"))
chart-xml (structured->chart-xml structured (:chart-id chart-result) chart-type)]
- {:output (str "<result>\n" chart-xml "\n</result>\n"
- "<instructions>\n" instruction-text "\n</instructions>")
+ {:output (tool-output chart-xml instruction-text)
:data-parts [(streaming/viz-part
{:entity-id (:chart-id chart-result)
:query-id (:query-id structured)
@@ -890,14 +932,6 @@
:title title
:description description})]
:structured-output full-structured
- :instructions instruction-text})
- ;; query-result may already have :output (error) or only :structured-output
- (if-let [s (or (:structured-output query-result) (:structured_output query-result))]
- (let [query-xml (llm-shape/query->xml (structured->query-data s))
- instruction-text (instructions/query-created-instructions-for (:query-id s) false)]
- (assoc query-result
- :output (str "<result>\n" query-xml "\n</result>\n"
- "<instructions>\n" instruction-text "\n</instructions>")))
- query-result)))
+ :instructions instruction-text})))
(catch Exception e
(tools.u/handle-agent-or-api-error e))))
diff --git a/test/metabase/metabot/agent/core_test.clj b/test/metabase/metabot/agent/core_test.clj
index 9ee68561d18..84a5d8f1b78 100644
--- a/test/metabase/metabot/agent/core_test.clj
+++ b/test/metabase/metabot/agent/core_test.clj
@@ -158,9 +158,9 @@
(is (contains? tools "replace_sql_query")))))))
(defn- run-query-offered-at-turn-start
- "Whether the turn starts with `run_query` among its tools, and whether its system message says results can be read.
- Reads them where the decision is made rather than running a turn. The bindings stand in for the ones the agent
- loop makes."
+ "Whether the turn starts with `run_query` among its tools, whether its system message says results can be read, and
+ whether it declares `show_chart` on construct_notebook_query. Reads them where the decision is made rather than
+ running a turn. The bindings stand in for the ones the agent loop makes."
[conversation-id capabilities]
(let [{:keys [tools system-msg]}
(binding [scope/*current-user-metabot-permissions* scope/all-yes-permissions
@@ -169,8 +169,13 @@
:profile-id :internal
:conversation-id conversation-id
:context {:capabilities capabilities}}))]
- {:tool? (contains? tools "run_query")
- :prompt? (str/includes? (pr-str system-msg) "You can see results only")}))
+ {:tool? (contains? tools "run_query")
+ :prompt? (str/includes? (pr-str system-msg) "You can see results only")
+ :show-chart? (-> (get tools "construct_notebook_query")
+ :declaration
+ deref
+ (get-in [:parameters :properties])
+ (contains? :show_chart))}))
(deftest client-claimed-feature-capability-cannot-override-settings-test
;; In a conversation the user owns, so the setting decides and not the rule for shared conversations.
@@ -191,15 +196,15 @@
:model/MetabotConversation {someone :id} {:user_id (mt/user->id :crowberto)}]
(mt/with-temporary-setting-values [metabot-query-execution-enabled? true]
(mt/with-current-user rasta
- (testing "the user's own conversation offers run_query and says results can be read"
- (is (= {:tool? true, :prompt? true} (offered own))))
- (testing "a conversation other people can read offers neither"
+ (testing "the user's own conversation offers run_query, says results can be read, and declares show_chart"
+ (is (= {:tool? true, :prompt? true, :show-chart? true} (offered own))))
+ (testing "a conversation other people can read does none of them"
(testing "a Slack thread"
- (is (= {:tool? false, :prompt? false} (offered thread))))
+ (is (= {:tool? false, :prompt? false, :show-chart? false} (offered thread))))
(testing "a conversation someone else started"
- (is (= {:tool? false, :prompt? false} (offered someone)))))
- (testing "no conversation offers neither"
- (is (= {:tool? false, :prompt? false} (offered nil)))))))))
+ (is (= {:tool? false, :prompt? false, :show-chart? false} (offered someone)))))
+ (testing "no conversation does none of them"
+ (is (= {:tool? false, :prompt? false, :show-chart? false} (offered nil)))))))))
(deftest document-sql-chart-tool-requires-native-permission-test
(mt/with-no-data-perms-for-all-users!
diff --git a/test/metabase/metabot/agent/prompt_gating_test.clj b/test/metabase/metabot/agent/prompt_gating_test.clj
index a54c889bc6c..ee5a19522be 100644
--- a/test/metabase/metabot/agent/prompt_gating_test.clj
+++ b/test/metabase/metabot/agent/prompt_gating_test.clj
@@ -323,6 +323,7 @@
(let [rendered (render-internal-template all-yes-perms ["construct_notebook_query" "run_query"])]
(is (re-find #"# You can see results only by running a query" rendered))
(is (re-find #"To read a saved question or model.*`source-card`" rendered))
+ (is (re-find #"pass `show_chart: false` to `construct_notebook_query`" rendered))
(is (not (re-find #"you cannot see query results" rendered))))))
(deftest prompt-gates-sql-execution-guidance-test
diff --git a/test/metabase/metabot/tools_test.clj b/test/metabase/metabot/tools_test.clj
index 0c8f757c538..5f4fbaeadcd 100644
--- a/test/metabase/metabot/tools_test.clj
+++ b/test/metabase/metabot/tools_test.clj
@@ -189,6 +189,40 @@
(is (= "Total order count."
(get-in result [:data-parts 0 :data :description]))))))))
+(deftest construct-notebook-query-tool-show-chart-test
+ (let [construct! (fn [tool-names show-chart-arg]
+ (let [chart-created? (atom false)]
+ (mt/with-dynamic-fn-redefs [construct/execute-representations-query
+ (fn [_ _]
+ {:structured-output {:query-id "q-1"
+ :query {:database 1}
+ :result-columns []}})
+ create-chart-tools/create-chart
+ (fn [_]
+ (reset! chart-created? true)
+ {:chart-id "c-1", :chart-type :table})]
+ (let [result (binding [shared/*profile-id* :nlq
+ shared/*memory-atom* (atom {:tool-names tool-names})]
+ (agent-tools/construct-notebook-query-tool
+ (merge {:query {:lib/type "mbql/query", :stages []}
+ :title "Seat check"
+ :description "Total order count."}
+ show-chart-arg)))]
+ {:chart? @chart-created?
+ :shown? (boolean (seq (:data-parts result)))
+ :query-id (get-in result [:structured-output :query-id])
+ :run-it? (boolean (re-find #"call `run_query` with `query_id` `q-1`"
+ (:output result)))}))))]
+ (testing "with run_query offered, the query is built and the user is shown nothing"
+ (is (= {:chart? false, :shown? false, :query-id "q-1", :run-it? true}
+ (construct! #{"construct_notebook_query" "run_query"} {:show_chart false}))))
+ (testing "with run_query offered and the argument left out, the chart is shown"
+ (is (= {:chart? true, :shown? true, :query-id "q-1", :run-it? false}
+ (construct! #{"construct_notebook_query" "run_query"} {}))))
+ (testing "without run_query the chart is shown whatever the argument says"
+ (is (= {:chart? true, :shown? true, :query-id "q-1", :run-it? false}
+ (construct! #{"construct_notebook_query"} {:show_chart false}))))))
+
(defn- construct-tool-output-for-thrown
"Run `construct_notebook_query` with `execute-representations-query` throwing `e`.
Returns the `:output` the LLM would see when the tool handles `e`; otherwise `e` propagates."