The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Stop Metabot writing SQL for the Internal Metabase Database (#86123)
Andrei Simionescu0ae314ca742a
diff --git a/enterprise/backend/test/metabase_enterprise/metabot/tools/sql_test.clj b/enterprise/backend/test/metabase_enterprise/metabot/tools/sql_test.cljindex 05a4e0383e4..75b5547c5a5 100644--- a/enterprise/backend/test/metabase_enterprise/metabot/tools/sql_test.clj+++ b/enterprise/backend/test/metabase_enterprise/metabot/tools/sql_test.clj@@ -1,22 +1,33 @@(ns metabase-enterprise.metabot.tools.sql-test(:require+ [clojure.string :as str][clojure.test :refer [deftest is testing use-fixtures]][metabase-enterprise.audit-app.audit-test :as audit-test][metabase-enterprise.sandbox.test-util :as sandbox.tu][metabase.api.common :as api][metabase.audit-app.core :as audit]+ [metabase.lib-be.core :as lib-be][metabase.lib.core :as lib]+ [metabase.metabot.agent.core-test :as agent-test][metabase.metabot.scope :as scope]+ [metabase.metabot.test-util :as test-util][metabase.metabot.tools :as tools]+ [metabase.metabot.tools.resources :as read-resource][metabase.metabot.tools.run-query :as run-query]+ [metabase.metabot.tools.sql :as sql]+ [metabase.metabot.tools.sql-test :as sql-test][metabase.metabot.tools.sql.common :as sql.common][metabase.permissions.core :as perms][metabase.query-processor :as qp][metabase.test :as mt]- [metabase.test.fixtures :as fixtures]))+ [metabase.test.fixtures :as fixtures]+ [toucan2.core :as t2]))(use-fixtures :once (fixtures/initialize :db :plugins))+(def ^:private audit-sql-unavailable-message+ "Metabase does not run SQL on the Internal Metabase Database, for any user.")+(deftest sandboxed-user-has-no-native-query-access-test(sandbox.tu/with-gtaps! {:gtaps {:checkins {}}};; The permissions API refuses SQL access to a sandboxed group now, but instances can still hold it from before.@@ -27,6 +38,91 @@(qp/process-query (lib/native-query (mt/metadata-provider) "SELECT 1"))))(is (false? (sql.common/native-query-access? (mt/id))))))+(deftest sql-tools-refuse-the-audit-database-test+ (mt/with-premium-features #{:audit-app}+ (audit-test/with-audit-db-restoration!+ (mt/with-db-perm-for-group! (perms/all-users-group) audit/audit-db-id :perms/create-queries :query-builder+ (let [query (lib/native-query (lib-be/application-database-metadata-provider audit/audit-db-id) "SELECT 1")+ memory (assoc @#'sql-test/editor-memory :state {:queries {"q1" query}})+ args {:database_id audit/audit-db-id+ :edits [{:old_string "1" :new_string "2"}]+ :new_query "SELECT 2"}]+ (doseq [user [:crowberto :rasta]+ profile [:internal :sql :internal-sql]+ tool-name ["create_sql_query" "edit_sql_query" "replace_sql_query"]]+ (testing (str user " " profile " " tool-name)+ (mt/with-current-user (mt/user->id user)+ (let [{:keys [error output]} (#'sql-test/invoke-sql-tool profile memory tool-name args)]+ (is (= (cond-> {:class (if (= :internal profile) :recoverable :unrecoverable)+ :code (if (= :internal profile)+ ::sql/sql-on-audit-database-with-notebook-fallback+ ::sql.common/sql-on-audit-database)}+ (not= :internal profile)+ (assoc :user-message audit-sql-unavailable-message))+ error))+ (is (str/includes? (or (:user-message error) output)+ "Metabase does not run SQL on the Internal Metabase Database"))+ (is (= (= :internal profile)+ (str/includes? output "Build the question with `construct_notebook_query`"))))))))))))++(deftest audit-sql-refusal-ends-a-turn-without-the-notebook-builder-test+ (mt/with-premium-features #{:audit-app}+ (audit-test/with-audit-db-restoration!+ (mt/with-db-perm-for-group! (perms/all-users-group) (mt/id) :perms/create-queries :query-builder-and-native+ (mt/with-db-perm-for-group! (perms/all-users-group) audit/audit-db-id :perms/create-queries :query-builder+ (mt/with-temp-env-var-value! [mb-metabot-eval-profiles-enabled true]+ (doseq [user [:crowberto :rasta]+ profile [:sql :internal-sql :document-generate-content]]+ (testing (str user " " profile)+ (mt/with-current-user (mt/user->id user)+ (let [{:keys [llm-calls parts]}+ (if (= :document-generate-content profile)+ (#'agent-test/run-turn!+ profile "Query Usage analytics"+ (mapv (fn [n]+ [{:type :tool-input :id (str "retry-" n)+ :function "document_construct_sql_chart"+ :arguments {:database_id audit/audit-db-id+ :name "Birds" :description "Birds" :analysis "Birds" :approach "SQL"+ :sql "SELECT 1" :viz_settings {:chart_type "table"}}}])+ (range 21))+ :context {:capabilities ["permission:write_sql_queries"]})+ (#'agent-test/run-sql-turn!+ profile audit/audit-db-id+ :calls (mapv #(vector (str "retry-" %)) (range 21))))]+ (is (= 1 llm-calls))+ (is (= :terminal-error (:finish-reason (last parts))))+ (is (some #(and (= :text (:type %))+ (= audit-sql-unavailable-message (:text %)))+ parts))))))))))))++(deftest chat-recovers-from-audit-sql-refusal-in-the-same-turn-test+ (mt/with-premium-features #{:audit-app}+ (audit-test/with-audit-db-restoration!+ (mt/with-db-perm-for-group! (perms/all-users-group) (mt/id) :perms/create-queries :query-builder-and-native+ (mt/with-db-perm-for-group! (perms/all-users-group) audit/audit-db-id :perms/create-queries :query-builder+ (doseq [user [:crowberto :rasta]]+ (testing user+ (mt/with-current-user (mt/user->id user)+ (let [{:keys [llm-calls parts]} (#'agent-test/run-sql-turn! :internal audit/audit-db-id)]+ (is (= 2 llm-calls))+ (is (= :stop (:finish-reason (last parts))))+ (is (=? {:type :tool-output+ :result {:error {:class :recoverable+ :code ::sql/sql-on-audit-database-with-notebook-fallback}+ :output #"(?s).*Build the question with `construct_notebook_query`.*"}}+ (some #(when (= :tool-output (:type %)) %) parts))))))))))))++(deftest read-resource-says-sql-is-unavailable-on-the-audit-database-test+ (mt/with-premium-features #{:audit-app}+ (audit-test/with-audit-db-restoration!+ (mt/with-current-user (mt/user->id :crowberto)+ (let [model-id (t2/select-one-pk :model/Card :database_id audit/audit-db-id :type :model)]+ (doseq [uri [(str "metabase://database/" audit/audit-db-id) (str "metabase://model/" model-id)]]+ (testing uri+ (is (str/includes? (:output (test-util/call-tool read-resource/read-resource-tool {:uris [uri]}))+ "sql_unavailable=\"true\"")))))))))+(defn- call-tool!"Call the tool `tool-name` of `tool-vars` with `args` as the current user, in a conversation of their own thatholds `queries`, with all Metabot permissions. Return the tool's result, or@@ -63,7 +159,7 @@(testing "and does not call it a permission gap"(is (re-find #"is the audit database" (:output result)))(is (not (re-find #"(?i)you do not have permission" (:output result)))))))- (testing "a SQL tool with no notebook fallback gets the QP's refusal, and the turn does not end"+ (testing "a SQL tool with no notebook fallback refuses creation and ends the turn"(let [result (call-tool! [#'tools/document-construct-sql-chart-tool]{}"document_construct_sql_chart"@@ -74,9 +170,10 @@:approach "SQL":sql "SELECT * FROM v_audit_log":viz_settings {:chart_type "table"}})]- (is (= {:class :recoverable, :code ::sql.common/query-failed}+ (is (= {:class :unrecoverable, :code ::sql.common/sql-on-audit-database+ :user-message audit-sql-unavailable-message}(:error result)))- (is (re-find #"Native queries are not allowed on the audit database" (:output result)))))+ (is (= audit-sql-unavailable-message (get-in result [:error :user-message])))))(testing "the QP's refusal of a table that is not an audit view reaches the model"(mt/with-temp [:model/Table table {:db_id audit/audit-db-id}:model/Field _ {:table_id (:id table)}]diff --git a/resources/metabot/prompts/llm_shape.selmer b/resources/metabot/prompts/llm_shape.selmerindex b31df5d626f..a22ca31ae95 100644--- a/resources/metabot/prompts/llm_shape.selmer+++ b/resources/metabot/prompts/llm_shape.selmer@@ -64,7 +64,7 @@ This will help avoid errors due to unexpected data types or formats.</metric>{% endif %}{% if is_table %}-<table id="{{table_id}}", name="{{table_name}}" database_id="{{table_database_id}}"{% if table_database_name %} database_name="{{table_database_name}}"{% endif %} database_engine="{{table_database_engine}}" fully_qualified_name="{{table_fqn}}">+<table id="{{table_id}}", name="{{table_name}}" database_id="{{table_database_id}}"{% if table_database_name %} database_name="{{table_database_name}}"{% endif %} database_engine="{{table_database_engine}}"{% if table_sql_unavailable %} sql_unavailable="true"{% endif %} fully_qualified_name="{{table_fqn}}">### Description{{table_description}}@@ -108,7 +108,7 @@ This will help avoid errors due to unexpected data types or formats.</table>{% endif %}{% if is_model %}-<metabase-model id="{{model_id}}" name="{{model_name}}" is_verified="{{model_verified}}" database_id="{{model_database_id}}"{% if model_database_name %} database_name="{{model_database_name}}"{% endif %} database_engine="{{model_database_engine}}" fully_qualified_name="{{model_fqn}}"{% if model_portable_entity_id %} portable_entity_id="{{model_portable_entity_id}}"{% endif %}>+<metabase-model id="{{model_id}}" name="{{model_name}}" is_verified="{{model_verified}}" database_id="{{model_database_id}}"{% if model_database_name %} database_name="{{model_database_name}}"{% endif %} database_engine="{{model_database_engine}}"{% if model_sql_unavailable %} sql_unavailable="true"{% endif %} fully_qualified_name="{{model_fqn}}"{% if model_portable_entity_id %} portable_entity_id="{{model_portable_entity_id}}"{% endif %}>### Description{{model_description}}@@ -288,7 +288,7 @@ or jargon that may be relevant to the user's questions.</user>{% endif %}{% if is_search_result %}-<{{search_tag_name}} id="{{search_id}}"{% if search_uri %} uri="{{search_uri}}"{% endif %} name="{{search_name}}"{% if search_has_curated %} is_curated="{{search_curated}}"{% endif %}{% if search_has_verified %} is_verified="{{search_verified}}"{% endif %}{% if search_has_official %} is_official="{{search_official}}"{% endif %}{% if search_has_can_write %} can_write="{{search_can_write}}"{% endif %}{% if search_data_layer %} data_layer="{{search_data_layer}}"{% endif %}{% if search_data_authority %} data_authority="{{search_data_authority}}"{% endif %}{% if search_database_id %} database_id="{{search_database_id}}"{% endif %}{% if search_database_name %} database_name="{{search_database_name}}"{% endif %}{% if search_database_engine %} database_engine="{{search_database_engine}}"{% endif %}{% if search_fqn %} fully_qualified_name="{{search_fqn}}"{% endif %}{% if search_base_table_fqn %} base_table_fully_qualified_name="{{search_base_table_fqn}}"{% endif %}{% if search_source_card_entity_id %} source_card_portable_entity_id="{{search_source_card_entity_id}}"{% if search_source_card_name %} source_card_name="{{search_source_card_name}}"{% endif %}{% endif %}{% if search_source_unavailable %} source_unavailable="true"{% endif %}{% if search_portable_entity_id %} portable_entity_id="{{search_portable_entity_id}}"{% endif %}>+<{{search_tag_name}} id="{{search_id}}"{% if search_uri %} uri="{{search_uri}}"{% endif %} name="{{search_name}}"{% if search_has_curated %} is_curated="{{search_curated}}"{% endif %}{% if search_has_verified %} is_verified="{{search_verified}}"{% endif %}{% if search_has_official %} is_official="{{search_official}}"{% endif %}{% if search_has_can_write %} can_write="{{search_can_write}}"{% endif %}{% if search_data_layer %} data_layer="{{search_data_layer}}"{% endif %}{% if search_data_authority %} data_authority="{{search_data_authority}}"{% endif %}{% if search_database_id %} database_id="{{search_database_id}}"{% endif %}{% if search_database_name %} database_name="{{search_database_name}}"{% endif %}{% if search_database_engine %} database_engine="{{search_database_engine}}"{% endif %}{% if search_sql_unavailable %} sql_unavailable="true"{% endif %}{% if search_fqn %} fully_qualified_name="{{search_fqn}}"{% endif %}{% if search_base_table_fqn %} base_table_fully_qualified_name="{{search_base_table_fqn}}"{% endif %}{% if search_source_card_entity_id %} source_card_portable_entity_id="{{search_source_card_entity_id}}"{% if search_source_card_name %} source_card_name="{{search_source_card_name}}"{% endif %}{% endif %}{% if search_source_unavailable %} source_unavailable="true"{% endif %}{% if search_portable_entity_id %} portable_entity_id="{{search_portable_entity_id}}"{% endif %}>{% if search_description %}{{search_description}}{% endif %}diff --git a/src/metabase/metabot/tools/resources.clj b/src/metabase/metabot/tools/resources.cljindex faaf1782374..05d3fdc5da3 100644--- a/src/metabase/metabot/tools/resources.clj+++ b/src/metabase/metabot/tools/resources.clj@@ -68,6 +68,7 @@[clojure.string :as str][metabase.activity-feed.core :as activity-feed][metabase.api.common :as api]+ [metabase.audit-app.core :as audit][metabase.collections.models.collection :as collection][metabase.documents.core :as documents][metabase.documents.prose-mirror :as prose-mirror]@@ -454,12 +455,13 @@(defn- present-database"Trim a database row to a token-frugal item map for list responses."[{:keys [id name engine description]}]- {:type "database"- :id id- :name name- :engine (some-> engine clojure.core/name)- :description description- :uri (llm-shape/metabase-uri :database id)})+ (cond-> {:type "database"+ :id id+ :name name+ :engine (some-> engine clojure.core/name)+ :description description+ :uri (llm-shape/metabase-uri :database id)}+ (= audit/audit-db-id id) (assoc :sql_unavailable true)))(defn- present-collection"Trim a collection row to an item map. `path-name` may be supplied if the caller pre-computed it."diff --git a/src/metabase/metabot/tools/shared/llm_shape.clj b/src/metabase/metabot/tools/shared/llm_shape.cljindex 1f39c9ef48f..f03c7720b73 100644--- a/src/metabase/metabot/tools/shared/llm_shape.clj+++ b/src/metabase/metabot/tools/shared/llm_shape.clj@@ -13,6 +13,7 @@(:require[clojure.string :as str][metabase.agent-lib.representations.resolve :as repr.resolve]+ [metabase.audit-app.core :as audit][metabase.lib-be.core :as lib-be][metabase.lib.schema :as lib.schema][metabase.metabot.agent.prompts :as prompts]@@ -521,7 +522,8 @@Matches Python Table.get_llm_representation exactly, except we additionally surface`database_name` as a tag attribute so the LLM has the human-readable DB name availablewithout a separate `read_resource` call — it's the first slot of every portable FK in- the representations-format `construct_notebook_query` tool."+ the representations-format `construct_notebook_query` tool. A table in the audit database+ also carries `sql_unavailable`, because Metabase runs no SQL there."[{:keys [id name database_id database_name database_engine database_schemadescription fields related_tables related_tables_totalrelated_tables_without_fields measures segments]}]@@ -533,6 +535,7 @@:table_database_id (str database_id):table_database_name database_name:table_database_engine (database-engine-or-unknown database_engine)+ :table_sql_unavailable (= audit/audit-db-id database_id):table_fqn fqn:table_description description:table_fields_xml (when (seq fields)@@ -559,7 +562,7 @@(defn model->xml"Format model for LLM consumption.Matches Python Model.get_llm_representation exactly, except we additionally surface- `database_name` as a tag attribute (see [[table->xml]] for the rationale).+ `database_name` and `sql_unavailable` as tag attributes (see [[table->xml]] for the rationale).When the caller supplies `:query_json` (the model's `dataset_query` exported through therepresentations resolver as a portable repr-form map), it is rendered inside the@@ -578,6 +581,7 @@:model_database_id (str database_id):model_database_name database_name:model_database_engine (database-engine-or-unknown database_engine)+ :model_sql_unavailable (= audit/audit-db-id database_id):model_fqn fqn:model_description description:model_portable_entity_id portable_entity_id@@ -860,7 +864,7 @@includes `portable_entity_id` so the LLM can paste it verbatim into `source-card:`without an extra `read_resource` call, and (when available) `database_name` — thehuman-readable name the LLM needs as the first slot of every portable FK in- `construct_notebook_query`.+ `construct_notebook_query`. A result in the audit database carries `sql_unavailable`, as in [[table->xml]].Metric results also identify the source to use with `[metric, {}, <portable_entity_id>]` in an aggregation.`base_table_fully_qualified_name` and `database_name` identify a table source; `source_card_portable_entity_id`@@ -905,6 +909,7 @@:search_database_id (when database_id (str database_id)):search_database_name database_name:search_database_engine engine+ :search_sql_unavailable (= audit/audit-db-id database_id):search_fqn fqn:search_portable_entity_id portable_entity_id:search_base_table_fqn (when (and (vector? base_table_portable_fk)@@ -1037,7 +1042,7 @@"Attributes rendered as XML attrs on each <item> element. Order matters for stable output."[:type :id :dashcard_id :tab_id :name :uri :database_id :collection_id :table_id :can_write:schema :display_name :authority_level :is_personal :path :location- :engine :timestamp])+ :engine :sql_unavailable :timestamp])(defn- item-attrs->xml[item]diff --git a/src/metabase/metabot/tools/sql.clj b/src/metabase/metabot/tools/sql.cljindex 18ebd6dadad..fed2a849b45 100644--- a/src/metabase/metabot/tools/sql.clj+++ b/src/metabase/metabot/tools/sql.clj@@ -68,23 +68,29 @@{:payload [:map {:closed true}]}[_]{:message "You do not have permission to write SQL against this database."- :recovery [{:uses #{notebook-tool-name}- :text (str "Build the question with `construct_notebook_query` instead. Don't write SQL against"- " this database again.")}]})+ :recovery [sql.common/notebook-instead-step]})++(tools/defrecoverable sql-on-audit-database-with-notebook-fallback!+ "The SQL targets the audit database, and the notebook builder is offered this turn."+ {:payload [:map {:closed true}]}+ [_]+ {:message "Metabase does not run SQL on the Internal Metabase Database, for any user."+ :recovery [sql.common/notebook-instead-step]})(defn- do-with-notebook-fallback- "Call `thunk`, rethrowing its native-query refusal as [[no-native-permission-with-notebook-fallback!]]- when the notebook builder is offered this turn.- Without the notebook builder the refusal still ends the turn."+ "Call `thunk`, making SQL refusals recoverable when the notebook builder is offered this turn."[ctx thunk];; A user who may write SQL on any database gets the SQL tools, so the agent can pick a database where they may not.;; The recoverable error lets it try the notebook builder instead of ending the turn.(try(thunk)(catch clojure.lang.ExceptionInfo e- (if (and (= ::sql.common/no-native-permission (get-in (ex-data e) [tools.error/error-key :code]))- (contains? (:tool-names ctx) notebook-tool-name))- (tools.error/throw-recoverable! ::no-native-permission-with-notebook-fallback {} {:cause e})+ (if-let [recovery-code (when (contains? (:tool-names ctx) notebook-tool-name)+ (case (get-in (ex-data e) [tools.error/error-key :code])+ ::sql.common/no-native-permission ::no-native-permission-with-notebook-fallback+ ::sql.common/sql-on-audit-database ::sql-on-audit-database-with-notebook-fallback+ nil))]+ (tools.error/throw-recoverable! recovery-code {} {:cause e})(throw e)))))(defn- query-viz-partdiff --git a/src/metabase/metabot/tools/sql/common.clj b/src/metabase/metabot/tools/sql/common.cljindex 15817719a0b..170479f3c9c 100644--- a/src/metabase/metabot/tools/sql/common.clj+++ b/src/metabase/metabot/tools/sql/common.clj@@ -13,6 +13,7 @@(:require[clojure.string :as str][metabase.api.common :as api]+ [metabase.audit-app.core :as audit][metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.metabot.tools.core :as tools]@@ -160,6 +161,11 @@:edit? (boolean edit?)}):else (sql-syntax-error! {:dialect dialect :error-message error-message}))))+(def notebook-instead-step+ "The recovery step after a refusal to write SQL against a database: build the question in the notebook instead."+ {:uses #{"construct_notebook_query"}+ :text "Build the question with `construct_notebook_query` instead. Don't write SQL against this database again."})+(defn native-query-access?"Whether the current user may write and run native queries against `database-id`.No nil-user escape, unlike `qp.perms/current-user-has-adhoc-native-query-perms?`, and a@@ -174,8 +180,11 @@:native {:query "SELECT 1"}})))(defn check-native-query-access!- "Throw an unrecoverable error unless the user may write native queries against `database-id`."+ "Throw an unrecoverable error for the audit database or a user without native query permission on `database-id`."[database-id]+ (when (= audit/audit-db-id database-id)+ (tools/unrecoverable! ::sql-on-audit-database+ {:user-message (tru "Metabase does not run SQL on the Internal Metabase Database, for any user.")}))(when-not (native-query-access? database-id)(tools/unrecoverable! ::no-native-permission{:user-message (tru "You do not have permission to write SQL queries against this database. Native query permissions are required.")diff --git a/test/metabase/metabot/tools/shared/llm_shape_test.clj b/test/metabase/metabot/tools/shared/llm_shape_test.cljindex 4606e7a226d..5bf41b000ed 100644--- a/test/metabase/metabot/tools/shared/llm_shape_test.clj+++ b/test/metabase/metabot/tools/shared/llm_shape_test.clj@@ -2,6 +2,7 @@(:require[clojure.string :as str][clojure.test :refer :all]+ [metabase.audit-app.core :as audit][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.metabot.tools.shared.llm-shape :as llm-shape]@@ -791,6 +792,15 @@:pages 1})"can_write=\"false\"")))+(deftest ^:parallel sql-unavailable-on-the-audit-database-test+ (doseq [[database-id unavailable?] [[audit/audit-db-id true] [1 false]]+ [element ->xml entity] [["table" llm-shape/table->xml {:id 10 :name "v_users"}]+ ["model" llm-shape/model->xml {:id 6 :name "People"}]+ ["search result" llm-shape/search-result->xml {:id 6 :type :model :name "People"}]]+ :let [xml (->xml (assoc entity :database_id database-id))]]+ (testing (str element " in database " database-id)+ (is (= unavailable? (str/includes? xml "sql_unavailable=\"true\""))))))+(deftest ^:parallel search-results->xml-test(testing "formats multiple search results"(let [results [{:id 1 :type :metric :name "Metric 1"}