The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Steer Metabot away from SQL a user can't run (#85609)
Chris Truter7f8238268c42
diff --git a/enterprise/backend/test/metabase_enterprise/metabot/tools/sql_test.clj b/enterprise/backend/test/metabase_enterprise/metabot/tools/sql_test.cljnew file mode 100644index 00000000000..aec774b5566--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/metabot/tools/sql_test.clj@@ -0,0 +1,19 @@+(ns metabase-enterprise.metabot.tools.sql-test+ (:require+ [clojure.test :refer [deftest is]]+ [metabase-enterprise.sandbox.test-util :as sandbox.tu]+ [metabase.lib.core :as lib]+ [metabase.metabot.tools.sql.common :as sql.common]+ [metabase.permissions.core :as perms]+ [metabase.query-processor :as qp]+ [metabase.test :as mt]))++(deftest sandboxed-user-has-no-native-query-access-test+ (sandbox.tu/with-gtaps! {:gtaps {:checkins {}}}+ ;; The permissions API refuses SQL access to a sandboxed group now, but instances can still hold it from before.+ (perms/set-database-permission! &group (mt/id) :perms/create-queries :query-builder-and-native)+ (is (= :query-builder-and-native+ (perms/full-database-permission-for-user (mt/user->id :rasta) :perms/create-queries (mt/id))))+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"You do not have permissions to run this query"+ (qp/process-query (lib/native-query (mt/metadata-provider) "SELECT 1"))))+ (is (false? (sql.common/native-query-access? (mt/id))))))diff --git a/src/metabase/metabot/tools/sql.clj b/src/metabase/metabot/tools/sql.cljindex 329e6259204..b37c9f72dfa 100644--- a/src/metabase/metabot/tools/sql.clj+++ b/src/metabase/metabot/tools/sql.clj@@ -6,6 +6,7 @@[metabase.metabot.scope :as scope][metabase.metabot.tmpl :as te][metabase.metabot.tools.core :as tools]+ [metabase.metabot.tools.error :as tools.error][metabase.metabot.tools.run-query :as tools.run-query][metabase.metabot.tools.shared :as shared][metabase.metabot.tools.shared.instructions :as instructions]@@ -60,6 +61,30 @@:mode "rewrite":value sql}))+(tools/defrecoverable no-native-permission!+ "The user may not write SQL against the query's database, but can build the question in the notebook."+ {:payload [:map {:closed true}]}+ [_]+ {:message "You do not have permission to write SQL against this database."+ :recovery [{:uses #{"construct_notebook_query"}+ :text (str "Build the question with `construct_notebook_query` instead. Don't write SQL against"+ " this database again.")}]})++(defn- with-notebook-fallback+ "Call `thunk`. When it refuses SQL on a database and the notebook builder is available this turn,+ rethrow the refusal as [[no-native-permission!]], so the agent builds the question there instead+ of ending the turn. The SQL tools are offered to a user who may write SQL on any database, so the+ agent can still pick one where they may not. Without the notebook builder the refusal ends the+ turn."+ [ctx thunk]+ (try+ (thunk)+ (catch clojure.lang.ExceptionInfo e+ (if (and (= ::sql.common/no-native-permission (get-in (ex-data e) [tools.error/error-key :code]))+ (contains? (:tool-names ctx) "construct_notebook_query"))+ (tools.error/throw-recoverable! ::no-native-permission {} {:cause e})+ (throw e)))))+(defn- query-viz-part[{:keys [query-id query]} title](streaming/viz-part {:entity-id (str (random-uuid))@@ -93,7 +118,8 @@(tools/unrecoverable! ::missing-editor{:user-message (tru "No active SQL editor was found. Open a SQL editor and try again.")})))action-result (sql.common/validated-sql-action!- (create-sql-query-tools/create-sql-query {:database-id database-id :sql sql-query}))+ (with-notebook-fallback+ ctx #(create-sql-query-tools/create-sql-query {:database-id database-id :sql sql-query}))){:keys [query-id query-content]} action-resultstructured (assoc action-result :result-type :query)instr (if code-edit?@@ -149,11 +175,12 @@:metabot/capabilities #{:permission-write-sql-queries}:args edit-sql-schema})(handle [_ {query-id :query_id :keys [edits checklist title]} ctx]- (sql-edit-result (edit-sql-query-tools/edit-sql-query- {:query-id query-id- :edits edits- :checklist checklist- :queries-state (shared/current-queries-state (:memory-atom ctx))})+ (sql-edit-result (with-notebook-fallback+ ctx #(edit-sql-query-tools/edit-sql-query+ {:query-id query-id+ :edits edits+ :checklist checklist+ :queries-state (shared/current-queries-state (:memory-atom ctx))}))title instructions/edit-sql-query-instructions-for ctx)))(def edit-sql-query-tool@@ -181,11 +208,12 @@:metabot/capabilities #{:permission-write-sql-queries}:args replace-sql-schema})(handle [_ {query-id :query_id new-query :new_query :keys [checklist title]} ctx]- (sql-edit-result (replace-sql-query-tools/replace-sql-query- {:query-id query-id- :sql new-query- :checklist checklist- :queries-state (shared/current-queries-state (:memory-atom ctx))})+ (sql-edit-result (with-notebook-fallback+ ctx #(replace-sql-query-tools/replace-sql-query+ {:query-id query-id+ :sql new-query+ :checklist checklist+ :queries-state (shared/current-queries-state (:memory-atom ctx))}))title instructions/replace-sql-query-instructions-for ctx)))(def replace-sql-query-tooldiff --git a/src/metabase/metabot/tools/sql/common.clj b/src/metabase/metabot/tools/sql/common.cljindex 32fe3e2260c..120c393ebb6 100644--- a/src/metabase/metabot/tools/sql/common.clj+++ b/src/metabase/metabot/tools/sql/common.clj@@ -21,7 +21,7 @@[metabase.metabot.tools.shared.instructions :as instructions][metabase.metabot.tools.sql.validation :as metabot.tools.sql.validation][metabase.metabot.util :as metabot.u]- [metabase.permissions.core :as perms]+ [metabase.query-permissions.core :as query-perms][metabase.util.i18n :refer [tru]][metabase.util.malli.registry :as mr][metabase.util.quoting :as quoting]))@@ -148,10 +148,12 @@non-integer `database-id` fails closed."[database-id](and (int? database-id)- (= :query-builder-and-native- (perms/full-database-permission-for-user api/*current-user-id*- :perms/create-queries- database-id))))+ (some? api/*current-user-id*)+ ;; The check the QP makes before it runs SQL. Native permission alone is not enough: the user also needs+ ;; unrestricted data access, and no sandbox on the database from any of their groups.+ (query-perms/can-run-query? {:database database-id+ :type :native+ :native {:query "SELECT 1"}})))(defn check-native-query-access!"Throw an unrecoverable error unless the user may write native queries against `database-id`."diff --git a/test/metabase/metabot/agent/core_test.clj b/test/metabase/metabot/agent/core_test.cljindex b03dfafe444..eed09ba9d3f 100644--- a/test/metabase/metabot/agent/core_test.clj+++ b/test/metabase/metabot/agent/core_test.clj@@ -373,7 +373,7 @@(perms/set-database-permission! (perms-group/all-users) builder-db :perms/create-queries :query-builder)(perms/set-database-permission! (perms-group/all-users) unreadable-db :perms/create-queries :no)(mt/with-current-user (mt/user->id :rasta)- (testing ":sql forbids the model from answering in text, so the loop stops on the denial"+ (testing ":sql has no notebook builder to fall back to, so the loop stops on the denial"(let [{:keys [llm-calls parts]} (run-sql-denial-turn! :sql builder-db)](is (= 1 llm-calls)"one call, not the profile's 20 iterations")@@ -384,19 +384,20 @@"the refusal is emitted as assistant text — a tool result is not rendered to the user")(is (some #(= :data (:type %)) parts)"state data part still closes the turn")))- (testing (str ":internal stops on the same denial. The class decides, not the profile: only "- "the user can grant this permission, so there is nothing for the agent to do "- "with the turn. This used to continue, and the model paraphrased the refusal "- "in its own words — nicer wording, bought with an extra LLM call, a profile "- "exception, and a model that might have said something else entirely.")+ (testing (str ":internal has the notebook builder, so the agent is told to build the question "+ "there and the turn continues")(let [{:keys [llm-calls parts]} (run-sql-denial-turn! :internal builder-db)]- (is (= 1 llm-calls)- "one call, where the profile gate used to allow a second")- (is (= :terminal-error (:finish-reason (last parts))))- (is (some #(and (= :text (:type %))- (str/includes? (:text %) "do not have permission"))- parts)- "the tool's own `:user-message`, which is authored and specific")))+ (is (= 2 llm-calls))+ (is (=? {:type :tool-output+ :result {:error {:class :recoverable}+ :output #"(?s).*Build the question with `construct_notebook_query`.*"}}+ (some #(when (= :tool-output (:type %)) %) parts)))+ (is (not= :terminal-error (:finish-reason (last parts))))))+ (testing ":internal-sql is :internal without the notebook builder, so the turn stops on the denial"+ (mt/with-temp-env-var-value! [mb-metabot-eval-profiles-enabled true]+ (let [{:keys [llm-calls parts]} (run-sql-denial-turn! :internal-sql builder-db)]+ (is (= 1 llm-calls))+ (is (= :terminal-error (:finish-reason (last parts)))))))(testing "a database the user cannot read at all stops the turn the same way"(let [{:keys [llm-calls parts]} (run-sql-denial-turn! :sql unreadable-db)](is (= 1 llm-calls)@@ -410,7 +411,8 @@(is (= :terminal-tool (:finish-reason (last parts))))))))))(deftest blank-replacement-after-edit-test- (mt/as-admin+ ;; A real user, not `mt/as-admin`: the SQL tools refuse a caller without a user id.+ (mt/with-current-user (mt/user->id :crowberto)(mt/with-temp [:model/Database {db-id :id} {:engine :h2}](mt/with-temporary-setting-values [llm-providers llm.tu/default-connectionsllm-metabot-provider test-provider]diff --git a/test/metabase/metabot/tools/sql_test.clj b/test/metabase/metabot/tools/sql_test.cljindex 3acb8627d76..296899ec78f 100644--- a/test/metabase/metabot/tools/sql_test.clj+++ b/test/metabase/metabot/tools/sql_test.clj@@ -25,11 +25,13 @@(defn- invoke-sql-tool[profile memory tool-name args](binding [scope/*current-user-scope* api-scope/unrestricted]- (let [entries (tools/->entries [#'tools/edit-sql-query-tool #'tools/replace-sql-query-tool- (if (= profile :sql)- #'tools/create-sql-query-code-edit-tool- #'tools/create-sql-query-tool)- #'tools/read-resource-tool]+ (let [entries (tools/->entries (cond-> [#'tools/edit-sql-query-tool #'tools/replace-sql-query-tool+ (if (= profile :sql)+ #'tools/create-sql-query-code-edit-tool+ #'tools/create-sql-query-tool)+ #'tools/read-resource-tool]+ ;; Like the real profiles: :internal has the notebook builder, :sql does not.+ (= profile :internal) (conj #'tools/construct-notebook-query-tool))(atom memory) nil profile)]((get-in entries [tool-name :fn])(merge {:title "Birds"}@@ -227,15 +229,21 @@(mt/with-current-user (mt/user->id :rasta)(mt/with-db db(let [query (lib/native-query (mt/metadata-provider) "SELECT 1")]- (doseq [tool-name ["edit_sql_query" "replace_sql_query"]]- (let [result (invoke-sql-tool :sql {:state {:queries {"q1" query}}} tool-name- {:edits [{:old_string "1" :new_string "2"}]- :new_query "SELECT 2"})]- (is (= {:class :unrecoverable- :code ::sql.common/no-native-permission- :user-message "You do not have permission to write SQL queries against this database. Native query permissions are required."}- (:error result)))- (is (str/includes? (:output result) "Don't retry"))))))))))+ (doseq [tool-name ["edit_sql_query" "replace_sql_query"]+ :let [invoke #(invoke-sql-tool % {:state {:queries {"q1" query}}} tool-name+ {:edits [{:old_string "1" :new_string "2"}]+ :new_query "SELECT 2"})]]+ (testing (str tool-name " without the notebook builder")+ (let [result (invoke :sql)]+ (is (= {:class :unrecoverable+ :code ::sql.common/no-native-permission+ :user-message "You do not have permission to write SQL queries against this database. Native query permissions are required."}+ (:error result)))+ (is (str/includes? (:output result) "Don't retry"))))+ (testing (str tool-name " with the notebook builder")+ (let [result (invoke :internal)]+ (is (= {:class :recoverable :code ::sql/no-native-permission} (:error result)))+ (is (str/includes? (:output result) "Build the question with `construct_notebook_query`")))))))))))(deftest missing-sql-editor-test(doseq [memory [nil {} {:context {:user_is_viewing [{:type "code_editor" :buffers []}]}}]]@@ -273,22 +281,29 @@(mt/with-temp [:model/Database {db-id :id} {:engine :h2}](mt/with-no-data-perms-for-all-users!(perms/set-database-permission! (perms-group/all-users) db-id :perms/view-data :unrestricted)- (doseq [[permission expected-error]- [[:query-builder-and-native nil]- [:query-builder {:class :unrecoverable- :code ::sql.common/no-native-permission- :user-message "You do not have permission to write SQL queries against this database. Native query permissions are required."}]- [:no {:class :unrecoverable- :code ::sql.create/database-access-denied- :user-message "You do not have access to this database."}]]]+ (doseq [[permission profile expected-error]+ [[:query-builder-and-native :internal nil]+ [:query-builder-and-native :sql nil]+ [:query-builder :internal {:class :recoverable+ :code ::sql/no-native-permission}]+ [:query-builder :sql {:class :unrecoverable+ :code ::sql.common/no-native-permission+ :user-message "You do not have permission to write SQL queries against this database. Native query permissions are required."}]+ [:no :internal {:class :unrecoverable+ :code ::sql.create/database-access-denied+ :user-message "You do not have access to this database."}]+ [:no :sql {:class :unrecoverable+ :code ::sql.create/database-access-denied+ :user-message "You do not have access to this database."}]]](perms/set-database-permission! (perms-group/all-users) db-id :perms/create-queries permission)(mt/with-current-user (mt/user->id :rasta)- (doseq [editor? [false true]]- (let [result (invoke-sql-tool (if editor? :sql :internal) editor-memory "create_sql_query" {:database_id db-id})]+ (testing (str permission " " profile)+ (let [result (invoke-sql-tool profile editor-memory "create_sql_query" {:database_id db-id})](is (= expected-error (:error result)))- (if expected-error- (is (str/includes? (:output result) "Don't retry"))- (is (some? (:structured-output result)))))))))))+ (case (:class expected-error)+ nil (is (some? (:structured-output result)))+ :recoverable (is (str/includes? (:output result) "Build the question with `construct_notebook_query`"))+ :unrecoverable (is (str/includes? (:output result) "Don't retry"))))))))))(deftest creation-delivers-query-to-the-profile-test(mt/with-current-user (mt/user->id :crowberto)