Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Hide stored query results while query execution is off (#84529)

Chris Truter88296d1f86a8
diff --git a/src/metabase/metabot/persistence.clj b/src/metabase/metabot/persistence.clj
index 82fcb6f00d3..e11fcf272f5 100644
--- a/src/metabase/metabot/persistence.clj
+++ b/src/metabase/metabot/persistence.clj
@@ -12,6 +12,7 @@
[metabase.metabot.schema.v2 :as schema.v2]
[metabase.metabot.self.core :as self.core]
[metabase.metabot.settings :as metabot.settings]
+ [metabase.metabot.tools :as metabot.tools]
[metabase.metabot.used-tables :as used-tables]
[metabase.util :as u]
[metabase.util.json :as json]
@@ -475,19 +476,33 @@
[conversation-id]
(:external_id (leaf-message conversation-id)))
+(def ^:private hidden-query-results
+ "Replayed in place of a stored query result while query execution is off."
+ "Results hidden: query execution is turned off.")
+
+(defn- hidden-query-results?
+ "Whether the stored output of tool `part` holds query results that must not reach the model.
+ Turning query execution off hides them without changing what is stored, so turning it on again restores them."
+ [part]
+ (and (not (metabot.settings/metabot-query-execution-enabled?))
+ (metabot.tools/query-result-tool? (schema.v2/tool-part-name part))))
+
(defn tool-part->llm-messages
"A stored v2 tool part → LLM history pair: the assistant tool call plus its
tool result. An unresolved (`input-available`) call is paired with a
synthetic interrupted result by default, so an aborted turn's partial content
stays replayable rather than being silently dropped. Pass
`:on-unresolved :skip` to drop it instead (Slack replays only completed tool
- calls and gets assistant text from its own thread copy)."
+ calls and gets assistant text from its own thread copy).
+ While query execution is off, a stored query result is replaced with a note that it is hidden."
([part] (tool-part->llm-messages part nil))
([part {:keys [on-unresolved]}]
(when (schema.v2/tool-part? part)
(let [{:keys [state input output errorText toolCallId]} part
content (case state
- "output-available" (str (if (map? output) (:output output) output))
+ "output-available" (if (hidden-query-results? part)
+ hidden-query-results
+ (str (if (map? output) (:output output) output)))
"output-error" (or errorText "Tool execution failed")
"input-available" (when-not (= on-unresolved :skip)
"Tool execution interrupted by user")
diff --git a/src/metabase/metabot/tools.clj b/src/metabase/metabot/tools.clj
index 9b2f7d664eb..7c33918b050 100644
--- a/src/metabase/metabot/tools.clj
+++ b/src/metabase/metabot/tools.clj
@@ -116,6 +116,22 @@
"replace_sql_query"
"construct_notebook_query"})
+(def ^:private query-result-tool-names
+ "Names of the tools that return query results: those that need `:feature-query-execution`."
+ (delay
+ (into #{}
+ (comp (filter #(or (tools.legacy/legacy-tool? %) (satisfies? tools.core/Tool @%)))
+ (map tools.legacy/declaration-of)
+ (filter #(contains? (:metabot/capabilities %) :feature-query-execution))
+ (map :name))
+ (vals (ns-publics 'metabase.metabot.tools)))))
+
+(defn query-result-tool?
+ "Whether the tool named `tool-name` returns query results, which may reach the model only while query execution
+ is on."
+ [tool-name]
+ (contains? @query-result-tool-names tool-name))
+
(defn ->entries
"The agent's tool entries for one turn: a map of model-facing tool name to entry.
diff --git a/test/metabase/metabot/persistence_test.clj b/test/metabase/metabot/persistence_test.clj
index f47296b351c..5f82ed43d9b 100644
--- a/test/metabase/metabot/persistence_test.clj
+++ b/test/metabase/metabot/persistence_test.clj
@@ -1486,6 +1486,30 @@
:error {:class :recoverable
:code :metabase.metabot.tools.recoverable.common/not-found}}}]))))))))
+(deftest history-hides-query-results-while-query-execution-is-off-test
+ (testing "a stored run_query result is replayed only while query execution is on, and nothing stored changes"
+ (with-rasta-tx
+ (let [conversation-id (str (random-uuid))
+ _ (seed-turn! conversation-id "how many?"
+ [{:type :tool-input :id "c1" :function "run_query" :arguments {:query_id "q"}}
+ {:type :tool-output :id "c1" :result {:output "<query_results>42</query_results>"}}
+ {:type :tool-input :id "c2" :function "search" :arguments {:q "x"}}
+ {:type :tool-output :id "c2" :result {:output "found"}}
+ {:type :text :text "There are 42."}])
+ tool-contents (fn []
+ (->> (metabot-persistence/history (metabot-persistence/live-messages conversation-id))
+ (filter #(= :tool (:role %)))
+ (mapv :content)))]
+ (is (= {:on ["<query_results>42</query_results>" "found"]
+ :off ["Results hidden: query execution is turned off." "found"]
+ :on-again ["<query_results>42</query_results>" "found"]}
+ {:on (mt/with-temporary-setting-values [metabot-query-execution-enabled? true]
+ (tool-contents))
+ :off (mt/with-temporary-setting-values [metabot-query-execution-enabled? false]
+ (tool-contents))
+ :on-again (mt/with-temporary-setting-values [metabot-query-execution-enabled? true]
+ (tool-contents))}))))))
+
(deftest history-drops-errored-turn-test
(testing "a turn whose reply errored is dropped whole; surrounding turns survive"
(with-rasta-tx