Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Steer Metabot away from SQL a user can't run (#85609)

Chris Truter7f8238268c42
diff --git a/enterprise/backend/test/metabase_enterprise/metabot/tools/sql_test.clj b/enterprise/backend/test/metabase_enterprise/metabot/tools/sql_test.clj
new file mode 100644
index 00000000000..aec774b5566
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/metabot/tools/sql_test.clj
@@ -0,0 +1,19 @@
+(ns metabase-enterprise.metabot.tools.sql-test
+ (:require
+ [clojure.test :refer [deftest is]]
+ [metabase-enterprise.sandbox.test-util :as sandbox.tu]
+ [metabase.lib.core :as lib]
+ [metabase.metabot.tools.sql.common :as sql.common]
+ [metabase.permissions.core :as perms]
+ [metabase.query-processor :as qp]
+ [metabase.test :as mt]))
+
+(deftest sandboxed-user-has-no-native-query-access-test
+ (sandbox.tu/with-gtaps! {:gtaps {:checkins {}}}
+ ;; The permissions API refuses SQL access to a sandboxed group now, but instances can still hold it from before.
+ (perms/set-database-permission! &group (mt/id) :perms/create-queries :query-builder-and-native)
+ (is (= :query-builder-and-native
+ (perms/full-database-permission-for-user (mt/user->id :rasta) :perms/create-queries (mt/id))))
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"You do not have permissions to run this query"
+ (qp/process-query (lib/native-query (mt/metadata-provider) "SELECT 1"))))
+ (is (false? (sql.common/native-query-access? (mt/id))))))
diff --git a/src/metabase/metabot/tools/sql.clj b/src/metabase/metabot/tools/sql.clj
index 329e6259204..b37c9f72dfa 100644
--- a/src/metabase/metabot/tools/sql.clj
+++ b/src/metabase/metabot/tools/sql.clj
@@ -6,6 +6,7 @@
[metabase.metabot.scope :as scope]
[metabase.metabot.tmpl :as te]
[metabase.metabot.tools.core :as tools]
+ [metabase.metabot.tools.error :as tools.error]
[metabase.metabot.tools.run-query :as tools.run-query]
[metabase.metabot.tools.shared :as shared]
[metabase.metabot.tools.shared.instructions :as instructions]
@@ -60,6 +61,30 @@
:mode "rewrite"
:value sql}))
+(tools/defrecoverable no-native-permission!
+ "The user may not write SQL against the query's database, but can build the question in the notebook."
+ {:payload [:map {:closed true}]}
+ [_]
+ {:message "You do not have permission to write SQL against this database."
+ :recovery [{:uses #{"construct_notebook_query"}
+ :text (str "Build the question with `construct_notebook_query` instead. Don't write SQL against"
+ " this database again.")}]})
+
+(defn- with-notebook-fallback
+ "Call `thunk`. When it refuses SQL on a database and the notebook builder is available this turn,
+ rethrow the refusal as [[no-native-permission!]], so the agent builds the question there instead
+ of ending the turn. The SQL tools are offered to a user who may write SQL on any database, so the
+ agent can still pick one where they may not. Without the notebook builder the refusal ends the
+ turn."
+ [ctx thunk]
+ (try
+ (thunk)
+ (catch clojure.lang.ExceptionInfo e
+ (if (and (= ::sql.common/no-native-permission (get-in (ex-data e) [tools.error/error-key :code]))
+ (contains? (:tool-names ctx) "construct_notebook_query"))
+ (tools.error/throw-recoverable! ::no-native-permission {} {:cause e})
+ (throw e)))))
+
(defn- query-viz-part
[{:keys [query-id query]} title]
(streaming/viz-part {:entity-id (str (random-uuid))
@@ -93,7 +118,8 @@
(tools/unrecoverable! ::missing-editor
{:user-message (tru "No active SQL editor was found. Open a SQL editor and try again.")})))
action-result (sql.common/validated-sql-action!
- (create-sql-query-tools/create-sql-query {:database-id database-id :sql sql-query}))
+ (with-notebook-fallback
+ ctx #(create-sql-query-tools/create-sql-query {:database-id database-id :sql sql-query})))
{:keys [query-id query-content]} action-result
structured (assoc action-result :result-type :query)
instr (if code-edit?
@@ -149,11 +175,12 @@
:metabot/capabilities #{:permission-write-sql-queries}
:args edit-sql-schema})
(handle [_ {query-id :query_id :keys [edits checklist title]} ctx]
- (sql-edit-result (edit-sql-query-tools/edit-sql-query
- {:query-id query-id
- :edits edits
- :checklist checklist
- :queries-state (shared/current-queries-state (:memory-atom ctx))})
+ (sql-edit-result (with-notebook-fallback
+ ctx #(edit-sql-query-tools/edit-sql-query
+ {:query-id query-id
+ :edits edits
+ :checklist checklist
+ :queries-state (shared/current-queries-state (:memory-atom ctx))}))
title instructions/edit-sql-query-instructions-for ctx)))
(def edit-sql-query-tool
@@ -181,11 +208,12 @@
:metabot/capabilities #{:permission-write-sql-queries}
:args replace-sql-schema})
(handle [_ {query-id :query_id new-query :new_query :keys [checklist title]} ctx]
- (sql-edit-result (replace-sql-query-tools/replace-sql-query
- {:query-id query-id
- :sql new-query
- :checklist checklist
- :queries-state (shared/current-queries-state (:memory-atom ctx))})
+ (sql-edit-result (with-notebook-fallback
+ ctx #(replace-sql-query-tools/replace-sql-query
+ {:query-id query-id
+ :sql new-query
+ :checklist checklist
+ :queries-state (shared/current-queries-state (:memory-atom ctx))}))
title instructions/replace-sql-query-instructions-for ctx)))
(def replace-sql-query-tool
diff --git a/src/metabase/metabot/tools/sql/common.clj b/src/metabase/metabot/tools/sql/common.clj
index 32fe3e2260c..120c393ebb6 100644
--- a/src/metabase/metabot/tools/sql/common.clj
+++ b/src/metabase/metabot/tools/sql/common.clj
@@ -21,7 +21,7 @@
[metabase.metabot.tools.shared.instructions :as instructions]
[metabase.metabot.tools.sql.validation :as metabot.tools.sql.validation]
[metabase.metabot.util :as metabot.u]
- [metabase.permissions.core :as perms]
+ [metabase.query-permissions.core :as query-perms]
[metabase.util.i18n :refer [tru]]
[metabase.util.malli.registry :as mr]
[metabase.util.quoting :as quoting]))
@@ -148,10 +148,12 @@
non-integer `database-id` fails closed."
[database-id]
(and (int? database-id)
- (= :query-builder-and-native
- (perms/full-database-permission-for-user api/*current-user-id*
- :perms/create-queries
- database-id))))
+ (some? api/*current-user-id*)
+ ;; The check the QP makes before it runs SQL. Native permission alone is not enough: the user also needs
+ ;; unrestricted data access, and no sandbox on the database from any of their groups.
+ (query-perms/can-run-query? {:database database-id
+ :type :native
+ :native {:query "SELECT 1"}})))
(defn check-native-query-access!
"Throw an unrecoverable error unless the user may write native queries against `database-id`."
diff --git a/test/metabase/metabot/agent/core_test.clj b/test/metabase/metabot/agent/core_test.clj
index b03dfafe444..eed09ba9d3f 100644
--- a/test/metabase/metabot/agent/core_test.clj
+++ b/test/metabase/metabot/agent/core_test.clj
@@ -373,7 +373,7 @@
(perms/set-database-permission! (perms-group/all-users) builder-db :perms/create-queries :query-builder)
(perms/set-database-permission! (perms-group/all-users) unreadable-db :perms/create-queries :no)
(mt/with-current-user (mt/user->id :rasta)
- (testing ":sql forbids the model from answering in text, so the loop stops on the denial"
+ (testing ":sql has no notebook builder to fall back to, so the loop stops on the denial"
(let [{:keys [llm-calls parts]} (run-sql-denial-turn! :sql builder-db)]
(is (= 1 llm-calls)
"one call, not the profile's 20 iterations")
@@ -384,19 +384,20 @@
"the refusal is emitted as assistant text — a tool result is not rendered to the user")
(is (some #(= :data (:type %)) parts)
"state data part still closes the turn")))
- (testing (str ":internal stops on the same denial. The class decides, not the profile: only "
- "the user can grant this permission, so there is nothing for the agent to do "
- "with the turn. This used to continue, and the model paraphrased the refusal "
- "in its own words — nicer wording, bought with an extra LLM call, a profile "
- "exception, and a model that might have said something else entirely.")
+ (testing (str ":internal has the notebook builder, so the agent is told to build the question "
+ "there and the turn continues")
(let [{:keys [llm-calls parts]} (run-sql-denial-turn! :internal builder-db)]
- (is (= 1 llm-calls)
- "one call, where the profile gate used to allow a second")
- (is (= :terminal-error (:finish-reason (last parts))))
- (is (some #(and (= :text (:type %))
- (str/includes? (:text %) "do not have permission"))
- parts)
- "the tool's own `:user-message`, which is authored and specific")))
+ (is (= 2 llm-calls))
+ (is (=? {:type :tool-output
+ :result {:error {:class :recoverable}
+ :output #"(?s).*Build the question with `construct_notebook_query`.*"}}
+ (some #(when (= :tool-output (:type %)) %) parts)))
+ (is (not= :terminal-error (:finish-reason (last parts))))))
+ (testing ":internal-sql is :internal without the notebook builder, so the turn stops on the denial"
+ (mt/with-temp-env-var-value! [mb-metabot-eval-profiles-enabled true]
+ (let [{:keys [llm-calls parts]} (run-sql-denial-turn! :internal-sql builder-db)]
+ (is (= 1 llm-calls))
+ (is (= :terminal-error (:finish-reason (last parts)))))))
(testing "a database the user cannot read at all stops the turn the same way"
(let [{:keys [llm-calls parts]} (run-sql-denial-turn! :sql unreadable-db)]
(is (= 1 llm-calls)
@@ -410,7 +411,8 @@
(is (= :terminal-tool (:finish-reason (last parts))))))))))
(deftest blank-replacement-after-edit-test
- (mt/as-admin
+ ;; A real user, not `mt/as-admin`: the SQL tools refuse a caller without a user id.
+ (mt/with-current-user (mt/user->id :crowberto)
(mt/with-temp [:model/Database {db-id :id} {:engine :h2}]
(mt/with-temporary-setting-values [llm-providers llm.tu/default-connections
llm-metabot-provider test-provider]
diff --git a/test/metabase/metabot/tools/sql_test.clj b/test/metabase/metabot/tools/sql_test.clj
index 3acb8627d76..296899ec78f 100644
--- a/test/metabase/metabot/tools/sql_test.clj
+++ b/test/metabase/metabot/tools/sql_test.clj
@@ -25,11 +25,13 @@
(defn- invoke-sql-tool
[profile memory tool-name args]
(binding [scope/*current-user-scope* api-scope/unrestricted]
- (let [entries (tools/->entries [#'tools/edit-sql-query-tool #'tools/replace-sql-query-tool
- (if (= profile :sql)
- #'tools/create-sql-query-code-edit-tool
- #'tools/create-sql-query-tool)
- #'tools/read-resource-tool]
+ (let [entries (tools/->entries (cond-> [#'tools/edit-sql-query-tool #'tools/replace-sql-query-tool
+ (if (= profile :sql)
+ #'tools/create-sql-query-code-edit-tool
+ #'tools/create-sql-query-tool)
+ #'tools/read-resource-tool]
+ ;; Like the real profiles: :internal has the notebook builder, :sql does not.
+ (= profile :internal) (conj #'tools/construct-notebook-query-tool))
(atom memory) nil profile)]
((get-in entries [tool-name :fn])
(merge {:title "Birds"}
@@ -227,15 +229,21 @@
(mt/with-current-user (mt/user->id :rasta)
(mt/with-db db
(let [query (lib/native-query (mt/metadata-provider) "SELECT 1")]
- (doseq [tool-name ["edit_sql_query" "replace_sql_query"]]
- (let [result (invoke-sql-tool :sql {:state {:queries {"q1" query}}} tool-name
- {:edits [{:old_string "1" :new_string "2"}]
- :new_query "SELECT 2"})]
- (is (= {:class :unrecoverable
- :code ::sql.common/no-native-permission
- :user-message "You do not have permission to write SQL queries against this database. Native query permissions are required."}
- (:error result)))
- (is (str/includes? (:output result) "Don't retry"))))))))))
+ (doseq [tool-name ["edit_sql_query" "replace_sql_query"]
+ :let [invoke #(invoke-sql-tool % {:state {:queries {"q1" query}}} tool-name
+ {:edits [{:old_string "1" :new_string "2"}]
+ :new_query "SELECT 2"})]]
+ (testing (str tool-name " without the notebook builder")
+ (let [result (invoke :sql)]
+ (is (= {:class :unrecoverable
+ :code ::sql.common/no-native-permission
+ :user-message "You do not have permission to write SQL queries against this database. Native query permissions are required."}
+ (:error result)))
+ (is (str/includes? (:output result) "Don't retry"))))
+ (testing (str tool-name " with the notebook builder")
+ (let [result (invoke :internal)]
+ (is (= {:class :recoverable :code ::sql/no-native-permission} (:error result)))
+ (is (str/includes? (:output result) "Build the question with `construct_notebook_query`")))))))))))
(deftest missing-sql-editor-test
(doseq [memory [nil {} {:context {:user_is_viewing [{:type "code_editor" :buffers []}]}}]]
@@ -273,22 +281,29 @@
(mt/with-temp [:model/Database {db-id :id} {:engine :h2}]
(mt/with-no-data-perms-for-all-users!
(perms/set-database-permission! (perms-group/all-users) db-id :perms/view-data :unrestricted)
- (doseq [[permission expected-error]
- [[:query-builder-and-native nil]
- [:query-builder {:class :unrecoverable
- :code ::sql.common/no-native-permission
- :user-message "You do not have permission to write SQL queries against this database. Native query permissions are required."}]
- [:no {:class :unrecoverable
- :code ::sql.create/database-access-denied
- :user-message "You do not have access to this database."}]]]
+ (doseq [[permission profile expected-error]
+ [[:query-builder-and-native :internal nil]
+ [:query-builder-and-native :sql nil]
+ [:query-builder :internal {:class :recoverable
+ :code ::sql/no-native-permission}]
+ [:query-builder :sql {:class :unrecoverable
+ :code ::sql.common/no-native-permission
+ :user-message "You do not have permission to write SQL queries against this database. Native query permissions are required."}]
+ [:no :internal {:class :unrecoverable
+ :code ::sql.create/database-access-denied
+ :user-message "You do not have access to this database."}]
+ [:no :sql {:class :unrecoverable
+ :code ::sql.create/database-access-denied
+ :user-message "You do not have access to this database."}]]]
(perms/set-database-permission! (perms-group/all-users) db-id :perms/create-queries permission)
(mt/with-current-user (mt/user->id :rasta)
- (doseq [editor? [false true]]
- (let [result (invoke-sql-tool (if editor? :sql :internal) editor-memory "create_sql_query" {:database_id db-id})]
+ (testing (str permission " " profile)
+ (let [result (invoke-sql-tool profile editor-memory "create_sql_query" {:database_id db-id})]
(is (= expected-error (:error result)))
- (if expected-error
- (is (str/includes? (:output result) "Don't retry"))
- (is (some? (:structured-output result)))))))))))
+ (case (:class expected-error)
+ nil (is (some? (:structured-output result)))
+ :recoverable (is (str/includes? (:output result) "Build the question with `construct_notebook_query`"))
+ :unrecoverable (is (str/includes? (:output result) "Don't retry"))))))))))
(deftest creation-delivers-query-to-the-profile-test
(mt/with-current-user (mt/user->id :crowberto)