Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Stop Metabot writing SQL for the Internal Metabase Database (#86123)

Andrei Simionescu0ae314ca742a
diff --git a/enterprise/backend/test/metabase_enterprise/metabot/tools/sql_test.clj b/enterprise/backend/test/metabase_enterprise/metabot/tools/sql_test.clj
index 05a4e0383e4..75b5547c5a5 100644
--- a/enterprise/backend/test/metabase_enterprise/metabot/tools/sql_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/metabot/tools/sql_test.clj
@@ -1,22 +1,33 @@
(ns metabase-enterprise.metabot.tools.sql-test
(:require
+ [clojure.string :as str]
[clojure.test :refer [deftest is testing use-fixtures]]
[metabase-enterprise.audit-app.audit-test :as audit-test]
[metabase-enterprise.sandbox.test-util :as sandbox.tu]
[metabase.api.common :as api]
[metabase.audit-app.core :as audit]
+ [metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
+ [metabase.metabot.agent.core-test :as agent-test]
[metabase.metabot.scope :as scope]
+ [metabase.metabot.test-util :as test-util]
[metabase.metabot.tools :as tools]
+ [metabase.metabot.tools.resources :as read-resource]
[metabase.metabot.tools.run-query :as run-query]
+ [metabase.metabot.tools.sql :as sql]
+ [metabase.metabot.tools.sql-test :as sql-test]
[metabase.metabot.tools.sql.common :as sql.common]
[metabase.permissions.core :as perms]
[metabase.query-processor :as qp]
[metabase.test :as mt]
- [metabase.test.fixtures :as fixtures]))
+ [metabase.test.fixtures :as fixtures]
+ [toucan2.core :as t2]))
(use-fixtures :once (fixtures/initialize :db :plugins))
+(def ^:private audit-sql-unavailable-message
+ "Metabase does not run SQL on the Internal Metabase Database, for any user.")
+
(deftest sandboxed-user-has-no-native-query-access-test
(sandbox.tu/with-gtaps! {:gtaps {:checkins {}}}
;; The permissions API refuses SQL access to a sandboxed group now, but instances can still hold it from before.
@@ -27,6 +38,91 @@
(qp/process-query (lib/native-query (mt/metadata-provider) "SELECT 1"))))
(is (false? (sql.common/native-query-access? (mt/id))))))
+(deftest sql-tools-refuse-the-audit-database-test
+ (mt/with-premium-features #{:audit-app}
+ (audit-test/with-audit-db-restoration!
+ (mt/with-db-perm-for-group! (perms/all-users-group) audit/audit-db-id :perms/create-queries :query-builder
+ (let [query (lib/native-query (lib-be/application-database-metadata-provider audit/audit-db-id) "SELECT 1")
+ memory (assoc @#'sql-test/editor-memory :state {:queries {"q1" query}})
+ args {:database_id audit/audit-db-id
+ :edits [{:old_string "1" :new_string "2"}]
+ :new_query "SELECT 2"}]
+ (doseq [user [:crowberto :rasta]
+ profile [:internal :sql :internal-sql]
+ tool-name ["create_sql_query" "edit_sql_query" "replace_sql_query"]]
+ (testing (str user " " profile " " tool-name)
+ (mt/with-current-user (mt/user->id user)
+ (let [{:keys [error output]} (#'sql-test/invoke-sql-tool profile memory tool-name args)]
+ (is (= (cond-> {:class (if (= :internal profile) :recoverable :unrecoverable)
+ :code (if (= :internal profile)
+ ::sql/sql-on-audit-database-with-notebook-fallback
+ ::sql.common/sql-on-audit-database)}
+ (not= :internal profile)
+ (assoc :user-message audit-sql-unavailable-message))
+ error))
+ (is (str/includes? (or (:user-message error) output)
+ "Metabase does not run SQL on the Internal Metabase Database"))
+ (is (= (= :internal profile)
+ (str/includes? output "Build the question with `construct_notebook_query`"))))))))))))
+
+(deftest audit-sql-refusal-ends-a-turn-without-the-notebook-builder-test
+ (mt/with-premium-features #{:audit-app}
+ (audit-test/with-audit-db-restoration!
+ (mt/with-db-perm-for-group! (perms/all-users-group) (mt/id) :perms/create-queries :query-builder-and-native
+ (mt/with-db-perm-for-group! (perms/all-users-group) audit/audit-db-id :perms/create-queries :query-builder
+ (mt/with-temp-env-var-value! [mb-metabot-eval-profiles-enabled true]
+ (doseq [user [:crowberto :rasta]
+ profile [:sql :internal-sql :document-generate-content]]
+ (testing (str user " " profile)
+ (mt/with-current-user (mt/user->id user)
+ (let [{:keys [llm-calls parts]}
+ (if (= :document-generate-content profile)
+ (#'agent-test/run-turn!
+ profile "Query Usage analytics"
+ (mapv (fn [n]
+ [{:type :tool-input :id (str "retry-" n)
+ :function "document_construct_sql_chart"
+ :arguments {:database_id audit/audit-db-id
+ :name "Birds" :description "Birds" :analysis "Birds" :approach "SQL"
+ :sql "SELECT 1" :viz_settings {:chart_type "table"}}}])
+ (range 21))
+ :context {:capabilities ["permission:write_sql_queries"]})
+ (#'agent-test/run-sql-turn!
+ profile audit/audit-db-id
+ :calls (mapv #(vector (str "retry-" %)) (range 21))))]
+ (is (= 1 llm-calls))
+ (is (= :terminal-error (:finish-reason (last parts))))
+ (is (some #(and (= :text (:type %))
+ (= audit-sql-unavailable-message (:text %)))
+ parts))))))))))))
+
+(deftest chat-recovers-from-audit-sql-refusal-in-the-same-turn-test
+ (mt/with-premium-features #{:audit-app}
+ (audit-test/with-audit-db-restoration!
+ (mt/with-db-perm-for-group! (perms/all-users-group) (mt/id) :perms/create-queries :query-builder-and-native
+ (mt/with-db-perm-for-group! (perms/all-users-group) audit/audit-db-id :perms/create-queries :query-builder
+ (doseq [user [:crowberto :rasta]]
+ (testing user
+ (mt/with-current-user (mt/user->id user)
+ (let [{:keys [llm-calls parts]} (#'agent-test/run-sql-turn! :internal audit/audit-db-id)]
+ (is (= 2 llm-calls))
+ (is (= :stop (:finish-reason (last parts))))
+ (is (=? {:type :tool-output
+ :result {:error {:class :recoverable
+ :code ::sql/sql-on-audit-database-with-notebook-fallback}
+ :output #"(?s).*Build the question with `construct_notebook_query`.*"}}
+ (some #(when (= :tool-output (:type %)) %) parts))))))))))))
+
+(deftest read-resource-says-sql-is-unavailable-on-the-audit-database-test
+ (mt/with-premium-features #{:audit-app}
+ (audit-test/with-audit-db-restoration!
+ (mt/with-current-user (mt/user->id :crowberto)
+ (let [model-id (t2/select-one-pk :model/Card :database_id audit/audit-db-id :type :model)]
+ (doseq [uri [(str "metabase://database/" audit/audit-db-id) (str "metabase://model/" model-id)]]
+ (testing uri
+ (is (str/includes? (:output (test-util/call-tool read-resource/read-resource-tool {:uris [uri]}))
+ "sql_unavailable=\"true\"")))))))))
+
(defn- call-tool!
"Call the tool `tool-name` of `tool-vars` with `args` as the current user, in a conversation of their own that
holds `queries`, with all Metabot permissions. Return the tool's result, or
@@ -63,7 +159,7 @@
(testing "and does not call it a permission gap"
(is (re-find #"is the audit database" (:output result)))
(is (not (re-find #"(?i)you do not have permission" (:output result)))))))
- (testing "a SQL tool with no notebook fallback gets the QP's refusal, and the turn does not end"
+ (testing "a SQL tool with no notebook fallback refuses creation and ends the turn"
(let [result (call-tool! [#'tools/document-construct-sql-chart-tool]
{}
"document_construct_sql_chart"
@@ -74,9 +170,10 @@
:approach "SQL"
:sql "SELECT * FROM v_audit_log"
:viz_settings {:chart_type "table"}})]
- (is (= {:class :recoverable, :code ::sql.common/query-failed}
+ (is (= {:class :unrecoverable, :code ::sql.common/sql-on-audit-database
+ :user-message audit-sql-unavailable-message}
(:error result)))
- (is (re-find #"Native queries are not allowed on the audit database" (:output result)))))
+ (is (= audit-sql-unavailable-message (get-in result [:error :user-message])))))
(testing "the QP's refusal of a table that is not an audit view reaches the model"
(mt/with-temp [:model/Table table {:db_id audit/audit-db-id}
:model/Field _ {:table_id (:id table)}]
diff --git a/resources/metabot/prompts/llm_shape.selmer b/resources/metabot/prompts/llm_shape.selmer
index b31df5d626f..a22ca31ae95 100644
--- a/resources/metabot/prompts/llm_shape.selmer
+++ b/resources/metabot/prompts/llm_shape.selmer
@@ -64,7 +64,7 @@ This will help avoid errors due to unexpected data types or formats.
</metric>
{% endif %}
{% if is_table %}
-<table id="{{table_id}}", name="{{table_name}}" database_id="{{table_database_id}}"{% if table_database_name %} database_name="{{table_database_name}}"{% endif %} database_engine="{{table_database_engine}}" fully_qualified_name="{{table_fqn}}">
+<table id="{{table_id}}", name="{{table_name}}" database_id="{{table_database_id}}"{% if table_database_name %} database_name="{{table_database_name}}"{% endif %} database_engine="{{table_database_engine}}"{% if table_sql_unavailable %} sql_unavailable="true"{% endif %} fully_qualified_name="{{table_fqn}}">
### Description
{{table_description}}
@@ -108,7 +108,7 @@ This will help avoid errors due to unexpected data types or formats.
</table>
{% endif %}
{% if is_model %}
-<metabase-model id="{{model_id}}" name="{{model_name}}" is_verified="{{model_verified}}" database_id="{{model_database_id}}"{% if model_database_name %} database_name="{{model_database_name}}"{% endif %} database_engine="{{model_database_engine}}" fully_qualified_name="{{model_fqn}}"{% if model_portable_entity_id %} portable_entity_id="{{model_portable_entity_id}}"{% endif %}>
+<metabase-model id="{{model_id}}" name="{{model_name}}" is_verified="{{model_verified}}" database_id="{{model_database_id}}"{% if model_database_name %} database_name="{{model_database_name}}"{% endif %} database_engine="{{model_database_engine}}"{% if model_sql_unavailable %} sql_unavailable="true"{% endif %} fully_qualified_name="{{model_fqn}}"{% if model_portable_entity_id %} portable_entity_id="{{model_portable_entity_id}}"{% endif %}>
### Description
{{model_description}}
@@ -288,7 +288,7 @@ or jargon that may be relevant to the user's questions.
</user>
{% endif %}
{% if is_search_result %}
-<{{search_tag_name}} id="{{search_id}}"{% if search_uri %} uri="{{search_uri}}"{% endif %} name="{{search_name}}"{% if search_has_curated %} is_curated="{{search_curated}}"{% endif %}{% if search_has_verified %} is_verified="{{search_verified}}"{% endif %}{% if search_has_official %} is_official="{{search_official}}"{% endif %}{% if search_has_can_write %} can_write="{{search_can_write}}"{% endif %}{% if search_data_layer %} data_layer="{{search_data_layer}}"{% endif %}{% if search_data_authority %} data_authority="{{search_data_authority}}"{% endif %}{% if search_database_id %} database_id="{{search_database_id}}"{% endif %}{% if search_database_name %} database_name="{{search_database_name}}"{% endif %}{% if search_database_engine %} database_engine="{{search_database_engine}}"{% endif %}{% if search_fqn %} fully_qualified_name="{{search_fqn}}"{% endif %}{% if search_base_table_fqn %} base_table_fully_qualified_name="{{search_base_table_fqn}}"{% endif %}{% if search_source_card_entity_id %} source_card_portable_entity_id="{{search_source_card_entity_id}}"{% if search_source_card_name %} source_card_name="{{search_source_card_name}}"{% endif %}{% endif %}{% if search_source_unavailable %} source_unavailable="true"{% endif %}{% if search_portable_entity_id %} portable_entity_id="{{search_portable_entity_id}}"{% endif %}>
+<{{search_tag_name}} id="{{search_id}}"{% if search_uri %} uri="{{search_uri}}"{% endif %} name="{{search_name}}"{% if search_has_curated %} is_curated="{{search_curated}}"{% endif %}{% if search_has_verified %} is_verified="{{search_verified}}"{% endif %}{% if search_has_official %} is_official="{{search_official}}"{% endif %}{% if search_has_can_write %} can_write="{{search_can_write}}"{% endif %}{% if search_data_layer %} data_layer="{{search_data_layer}}"{% endif %}{% if search_data_authority %} data_authority="{{search_data_authority}}"{% endif %}{% if search_database_id %} database_id="{{search_database_id}}"{% endif %}{% if search_database_name %} database_name="{{search_database_name}}"{% endif %}{% if search_database_engine %} database_engine="{{search_database_engine}}"{% endif %}{% if search_sql_unavailable %} sql_unavailable="true"{% endif %}{% if search_fqn %} fully_qualified_name="{{search_fqn}}"{% endif %}{% if search_base_table_fqn %} base_table_fully_qualified_name="{{search_base_table_fqn}}"{% endif %}{% if search_source_card_entity_id %} source_card_portable_entity_id="{{search_source_card_entity_id}}"{% if search_source_card_name %} source_card_name="{{search_source_card_name}}"{% endif %}{% endif %}{% if search_source_unavailable %} source_unavailable="true"{% endif %}{% if search_portable_entity_id %} portable_entity_id="{{search_portable_entity_id}}"{% endif %}>
{% if search_description %}
{{search_description}}
{% endif %}
diff --git a/src/metabase/metabot/tools/resources.clj b/src/metabase/metabot/tools/resources.clj
index faaf1782374..05d3fdc5da3 100644
--- a/src/metabase/metabot/tools/resources.clj
+++ b/src/metabase/metabot/tools/resources.clj
@@ -68,6 +68,7 @@
[clojure.string :as str]
[metabase.activity-feed.core :as activity-feed]
[metabase.api.common :as api]
+ [metabase.audit-app.core :as audit]
[metabase.collections.models.collection :as collection]
[metabase.documents.core :as documents]
[metabase.documents.prose-mirror :as prose-mirror]
@@ -454,12 +455,13 @@
(defn- present-database
"Trim a database row to a token-frugal item map for list responses."
[{:keys [id name engine description]}]
- {:type "database"
- :id id
- :name name
- :engine (some-> engine clojure.core/name)
- :description description
- :uri (llm-shape/metabase-uri :database id)})
+ (cond-> {:type "database"
+ :id id
+ :name name
+ :engine (some-> engine clojure.core/name)
+ :description description
+ :uri (llm-shape/metabase-uri :database id)}
+ (= audit/audit-db-id id) (assoc :sql_unavailable true)))
(defn- present-collection
"Trim a collection row to an item map. `path-name` may be supplied if the caller pre-computed it."
diff --git a/src/metabase/metabot/tools/shared/llm_shape.clj b/src/metabase/metabot/tools/shared/llm_shape.clj
index 1f39c9ef48f..f03c7720b73 100644
--- a/src/metabase/metabot/tools/shared/llm_shape.clj
+++ b/src/metabase/metabot/tools/shared/llm_shape.clj
@@ -13,6 +13,7 @@
(:require
[clojure.string :as str]
[metabase.agent-lib.representations.resolve :as repr.resolve]
+ [metabase.audit-app.core :as audit]
[metabase.lib-be.core :as lib-be]
[metabase.lib.schema :as lib.schema]
[metabase.metabot.agent.prompts :as prompts]
@@ -521,7 +522,8 @@
Matches Python Table.get_llm_representation exactly, except we additionally surface
`database_name` as a tag attribute so the LLM has the human-readable DB name available
without a separate `read_resource` call — it's the first slot of every portable FK in
- the representations-format `construct_notebook_query` tool."
+ the representations-format `construct_notebook_query` tool. A table in the audit database
+ also carries `sql_unavailable`, because Metabase runs no SQL there."
[{:keys [id name database_id database_name database_engine database_schema
description fields related_tables related_tables_total
related_tables_without_fields measures segments]}]
@@ -533,6 +535,7 @@
:table_database_id (str database_id)
:table_database_name database_name
:table_database_engine (database-engine-or-unknown database_engine)
+ :table_sql_unavailable (= audit/audit-db-id database_id)
:table_fqn fqn
:table_description description
:table_fields_xml (when (seq fields)
@@ -559,7 +562,7 @@
(defn model->xml
"Format model for LLM consumption.
Matches Python Model.get_llm_representation exactly, except we additionally surface
- `database_name` as a tag attribute (see [[table->xml]] for the rationale).
+ `database_name` and `sql_unavailable` as tag attributes (see [[table->xml]] for the rationale).
When the caller supplies `:query_json` (the model's `dataset_query` exported through the
representations resolver as a portable repr-form map), it is rendered inside the
@@ -578,6 +581,7 @@
:model_database_id (str database_id)
:model_database_name database_name
:model_database_engine (database-engine-or-unknown database_engine)
+ :model_sql_unavailable (= audit/audit-db-id database_id)
:model_fqn fqn
:model_description description
:model_portable_entity_id portable_entity_id
@@ -860,7 +864,7 @@
includes `portable_entity_id` so the LLM can paste it verbatim into `source-card:`
without an extra `read_resource` call, and (when available) `database_name` — the
human-readable name the LLM needs as the first slot of every portable FK in
- `construct_notebook_query`.
+ `construct_notebook_query`. A result in the audit database carries `sql_unavailable`, as in [[table->xml]].
Metric results also identify the source to use with `[metric, {}, <portable_entity_id>]` in an aggregation.
`base_table_fully_qualified_name` and `database_name` identify a table source; `source_card_portable_entity_id`
@@ -905,6 +909,7 @@
:search_database_id (when database_id (str database_id))
:search_database_name database_name
:search_database_engine engine
+ :search_sql_unavailable (= audit/audit-db-id database_id)
:search_fqn fqn
:search_portable_entity_id portable_entity_id
:search_base_table_fqn (when (and (vector? base_table_portable_fk)
@@ -1037,7 +1042,7 @@
"Attributes rendered as XML attrs on each <item> element. Order matters for stable output."
[:type :id :dashcard_id :tab_id :name :uri :database_id :collection_id :table_id :can_write
:schema :display_name :authority_level :is_personal :path :location
- :engine :timestamp])
+ :engine :sql_unavailable :timestamp])
(defn- item-attrs->xml
[item]
diff --git a/src/metabase/metabot/tools/sql.clj b/src/metabase/metabot/tools/sql.clj
index 18ebd6dadad..fed2a849b45 100644
--- a/src/metabase/metabot/tools/sql.clj
+++ b/src/metabase/metabot/tools/sql.clj
@@ -68,23 +68,29 @@
{:payload [:map {:closed true}]}
[_]
{:message "You do not have permission to write SQL against this database."
- :recovery [{:uses #{notebook-tool-name}
- :text (str "Build the question with `construct_notebook_query` instead. Don't write SQL against"
- " this database again.")}]})
+ :recovery [sql.common/notebook-instead-step]})
+
+(tools/defrecoverable sql-on-audit-database-with-notebook-fallback!
+ "The SQL targets the audit database, and the notebook builder is offered this turn."
+ {:payload [:map {:closed true}]}
+ [_]
+ {:message "Metabase does not run SQL on the Internal Metabase Database, for any user."
+ :recovery [sql.common/notebook-instead-step]})
(defn- do-with-notebook-fallback
- "Call `thunk`, rethrowing its native-query refusal as [[no-native-permission-with-notebook-fallback!]]
- when the notebook builder is offered this turn.
- Without the notebook builder the refusal still ends the turn."
+ "Call `thunk`, making SQL refusals recoverable when the notebook builder is offered this turn."
[ctx thunk]
;; A user who may write SQL on any database gets the SQL tools, so the agent can pick a database where they may not.
;; The recoverable error lets it try the notebook builder instead of ending the turn.
(try
(thunk)
(catch clojure.lang.ExceptionInfo e
- (if (and (= ::sql.common/no-native-permission (get-in (ex-data e) [tools.error/error-key :code]))
- (contains? (:tool-names ctx) notebook-tool-name))
- (tools.error/throw-recoverable! ::no-native-permission-with-notebook-fallback {} {:cause e})
+ (if-let [recovery-code (when (contains? (:tool-names ctx) notebook-tool-name)
+ (case (get-in (ex-data e) [tools.error/error-key :code])
+ ::sql.common/no-native-permission ::no-native-permission-with-notebook-fallback
+ ::sql.common/sql-on-audit-database ::sql-on-audit-database-with-notebook-fallback
+ nil))]
+ (tools.error/throw-recoverable! recovery-code {} {:cause e})
(throw e)))))
(defn- query-viz-part
diff --git a/src/metabase/metabot/tools/sql/common.clj b/src/metabase/metabot/tools/sql/common.clj
index 15817719a0b..170479f3c9c 100644
--- a/src/metabase/metabot/tools/sql/common.clj
+++ b/src/metabase/metabot/tools/sql/common.clj
@@ -13,6 +13,7 @@
(:require
[clojure.string :as str]
[metabase.api.common :as api]
+ [metabase.audit-app.core :as audit]
[metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
[metabase.metabot.tools.core :as tools]
@@ -160,6 +161,11 @@
:edit? (boolean edit?)})
:else (sql-syntax-error! {:dialect dialect :error-message error-message}))))
+(def notebook-instead-step
+ "The recovery step after a refusal to write SQL against a database: build the question in the notebook instead."
+ {:uses #{"construct_notebook_query"}
+ :text "Build the question with `construct_notebook_query` instead. Don't write SQL against this database again."})
+
(defn native-query-access?
"Whether the current user may write and run native queries against `database-id`.
No nil-user escape, unlike `qp.perms/current-user-has-adhoc-native-query-perms?`, and a
@@ -174,8 +180,11 @@
:native {:query "SELECT 1"}})))
(defn check-native-query-access!
- "Throw an unrecoverable error unless the user may write native queries against `database-id`."
+ "Throw an unrecoverable error for the audit database or a user without native query permission on `database-id`."
[database-id]
+ (when (= audit/audit-db-id database-id)
+ (tools/unrecoverable! ::sql-on-audit-database
+ {:user-message (tru "Metabase does not run SQL on the Internal Metabase Database, for any user.")}))
(when-not (native-query-access? database-id)
(tools/unrecoverable! ::no-native-permission
{:user-message (tru "You do not have permission to write SQL queries against this database. Native query permissions are required.")
diff --git a/test/metabase/metabot/tools/shared/llm_shape_test.clj b/test/metabase/metabot/tools/shared/llm_shape_test.clj
index 4606e7a226d..5bf41b000ed 100644
--- a/test/metabase/metabot/tools/shared/llm_shape_test.clj
+++ b/test/metabase/metabot/tools/shared/llm_shape_test.clj
@@ -2,6 +2,7 @@
(:require
[clojure.string :as str]
[clojure.test :refer :all]
+ [metabase.audit-app.core :as audit]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.metabot.tools.shared.llm-shape :as llm-shape]
@@ -791,6 +792,15 @@
:pages 1})
"can_write=\"false\"")))
+(deftest ^:parallel sql-unavailable-on-the-audit-database-test
+ (doseq [[database-id unavailable?] [[audit/audit-db-id true] [1 false]]
+ [element ->xml entity] [["table" llm-shape/table->xml {:id 10 :name "v_users"}]
+ ["model" llm-shape/model->xml {:id 6 :name "People"}]
+ ["search result" llm-shape/search-result->xml {:id 6 :type :model :name "People"}]]
+ :let [xml (->xml (assoc entity :database_id database-id))]]
+ (testing (str element " in database " database-id)
+ (is (= unavailable? (str/includes? xml "sql_unavailable=\"true\""))))))
+
(deftest ^:parallel search-results->xml-test
(testing "formats multiple search results"
(let [results [{:id 1 :type :metric :name "Metric 1"}