The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Remove the transform tools left behind by the transforms_codegen profile (#85817)
lbrdnk3d91fa0b1cb0
diff --git a/frontend/src/metabase/metabot/constants.ts b/frontend/src/metabase/metabot/constants.tsindex 9beced17f29..d784fbe6e84 100644--- a/frontend/src/metabase/metabot/constants.ts+++ b/frontend/src/metabase/metabot/constants.ts@@ -130,10 +130,6 @@ export const TOOL_MESSAGES = {active: () => t`Retrieving table metadata`,done: () => t`Retrieved table metadata`,},- get_transform_details: {- active: () => t`Getting transform details`,- done: () => t`Got transform details`,- },list_available_fields: { active: () => undefined, done: () => undefined },load_skill: { active: () => undefined, done: () => undefined },read_resource: {diff --git a/src/metabase/metabot/tools.clj b/src/metabase/metabot/tools.cljindex 5f51712b101..0fba7d6d0d4 100644--- a/src/metabase/metabot/tools.clj+++ b/src/metabase/metabot/tools.clj@@ -32,7 +32,6 @@[metabase.metabot.tools.subscriptions :as tools.subscriptions][metabase.metabot.tools.timelines :as tools.timelines][metabase.metabot.tools.todo :as tools.todo]- [metabase.metabot.tools.transforms :as tools.transforms][potemkin :as p]))(set! *warn-on-reflection* true)@@ -43,8 +42,7 @@analyze-chart-sql-tool][tools.search search-toolsql-search-tool- nlq-search-tool- transform-search-tool]+ nlq-search-tool][tools.entity-retrievalretrieve-library-entities-tool][tools.construct@@ -68,8 +66,6 @@list-available-data-sources-toollist-available-fields-toolget-field-values-tool]- [tools.transforms- get-transform-details-tool][tools.sqlcreate-sql-query-toolcreate-sql-query-code-edit-tooldiff --git a/src/metabase/metabot/tools/search.clj b/src/metabase/metabot/tools/search.cljindex c11df6d470c..015f5c13228 100644--- a/src/metabase/metabot/tools/search.clj+++ b/src/metabase/metabot/tools/search.clj@@ -11,7 +11,6 @@[metabase.metabot.search-models :as metabot.search-models][metabase.metabot.tmpl :as te][metabase.metabot.tools.core :as tools]- [metabase.metabot.tools.shared :as shared][metabase.metabot.tools.shared.instructions :as instructions][metabase.metabot.tools.shared.llm-shape :as llm-shape][metabase.metabot.tools.util :as metabot.tools.u]@@ -21,8 +20,7 @@[metabase.search.engine :as search.engine][metabase.transforms.core :as transforms][metabase.util :as u]- [metabase.util.log :as log]- [metabase.util.malli :as mu]))+ [metabase.util.log :as log]))(set! *warn-on-reflection* true)@@ -816,20 +814,3 @@(->SearchTool"Find NLQ-queryable data sources by topic, or find dashboards and documents as save destinations."nlq-search-schema ["metric" "measure" "segment" "model" "question" "table"] {:profile-id "nlq"}))--(def ^:private transform-search-schema- [:map {:closed true}- [:semantic_queries {:optional true :feature :semantic-search} semantic-queries-schema]- [:keyword_queries {:optional true} keyword-queries-schema]- [:search_native_query {:optional true} [:maybe [:boolean {:description "Also match against the native SQL text of transforms, not just names and descriptions."}]]]- [:entity_types {:optional true} (entity-types-schema "table" "model" "transform")]- [:limit {:optional true} limit-schema]])--(mu/defn ^{:tool-name "search"- :scope scope/agent-search- :title-fn search-display}- transform-search-tool- "Find transforms, plus the tables and models around them, by topic."- [{:keys [search_native_query] :as args} :- transform-search-schema]- (do-search (sorted-set "model" "table" "transform")- {:search-native-query search_native_query :metabot shared/*metabot*} args))diff --git a/src/metabase/metabot/tools/transforms.clj b/src/metabase/metabot/tools/transforms.cljdeleted file mode 100644index 8d6260a8c0b..00000000000--- a/src/metabase/metabot/tools/transforms.clj+++ /dev/null@@ -1,68 +0,0 @@-(ns metabase.metabot.tools.transforms- "Tools for reading transform definitions."- (:require- [clojure.string :as str]- [metabase.metabot.query-export :as query-export]- [metabase.metabot.scope :as scope]- [metabase.metabot.tools.shared.llm-shape :as llm-shape]- [metabase.metabot.tools.util :as metabot.tools.u]- [metabase.transforms.core :as transforms]- [metabase.util.malli :as mu]))--(set! *warn-on-reflection* true)--;;; ──────────────────────────────────────────────────────────────────-;;; Formatting helpers-;;; ──────────────────────────────────────────────────────────────────--;; Hand-built rather than `clojure.data.xml` so query and body content stays verbatim:-;; data.xml escapes `<`/`>`/`&`, and an escaped query breaks `old_string` matching when-;; the model quotes it back to the write tools.-(defn- format-transform-details-output- [{:keys [id description source target] :as transform}]- (let [{source-type :type, :keys [query body source-database]} source]- (->> [(str "<transform id=\"" id "\" name=\"" (llm-shape/escape-xml (:name transform)) "\">")- (when description- (str " <description>" (llm-shape/escape-xml-content description) "</description>"))- (when source- (str " <source type=\"" (some-> source-type name) "\">"))- (when-let [query-text (some-> query llm-shape/transform-query->text)]- (str " <query>" query-text "</query>"))- (when body- (str " <body>" body "</body>"))- (when source-database- (str " <database>" source-database "</database>"))- (when source- " </source>")- (when target- (str " <target>" (llm-shape/escape-xml-content (pr-str target)) "</target>"))- "</transform>"]- (remove nil?)- (str/join "\n"))))--(defn- add-output- "Add :output to a tool result. Handles both :structured_output and :structured-output."- [result format-fn]- (if-let [structured (or (:structured_output result) (:structured-output result))]- (assoc result :output (format-fn structured))- result))--;;; ──────────────────────────────────────────────────────────────────-;;; Tool definitions-;;; ──────────────────────────────────────────────────────────────────--(mu/defn ^{:tool-name "get_transform_details"- :scope scope/agent-transforms-read- :capabilities #{:feature-transforms}}- get-transform-details-tool- "Get information about a transform."- [{:keys [transform_id]} :- [:map {:closed true} [:transform_id :int]]]- (try- (add-output {:structured_output (query-export/transform-with-exportable-source- (transforms/get-transform transform_id))}- format-transform-details-output)- (catch Exception e- (if (= 403 (:status-code (ex-data e)))- ;; A permission refusal is an answer for the agent, not a tool failure -- relay the standard message.- {:output (ex-message e) :status-code 403}- (metabot.tools.u/handle-agent-error e)))))diff --git a/test/metabase/metabot/tools/search_test.clj b/test/metabase/metabot/tools/search_test.cljindex e4fe5ef3208..84e52d98a33 100644--- a/test/metabase/metabot/tools/search_test.clj+++ b/test/metabase/metabot/tools/search_test.clj@@ -692,8 +692,7 @@(testing "every search tool variant rejects a scalar the same way"(doseq [[tool-var extra-args] [[#'metabot.tools/search-tool {}][#'metabot.tools/sql-search-tool {:database_id 1}]- [#'metabot.tools/nlq-search-tool {}]- [#'metabot.tools/transform-search-tool {}]]]+ [#'metabot.tools/nlq-search-tool {}]]](testing (str tool-var)(is (= "Invalid tool arguments: `keyword_queries` must be an array of strings; received a string."(test-util/tool-boundary-error "search" tool-vardiff --git a/test/metabase/metabot/tools/transforms_test.clj b/test/metabase/metabot/tools/transforms_test.cljdeleted file mode 100644index f7c256c14af..00000000000--- a/test/metabase/metabot/tools/transforms_test.clj+++ /dev/null@@ -1,198 +0,0 @@-(ns metabase.metabot.tools.transforms-test- "Tests for reading transform definitions."- (:require- [clojure.string :as str]- [clojure.test :refer :all]- [metabase.lib.core :as lib]- [metabase.lib.metadata :as lib.metadata]- [metabase.metabot.metadata-perms :as metabot.perms]- [metabase.metabot.query-export :as query-export]- [metabase.metabot.tools.transforms :as agent-transforms]- [metabase.permissions.core :as perms]- [metabase.permissions.models.permissions-group :as perms-group]- [metabase.test :as mt]- [metabase.util.json :as json]- [toucan2.core :as t2]))--;;; ----------------------------------- read tool integration tests -----------------------------------------------------(def ^:private gadget-sql- "SELECT * FROM products WHERE price < 100 AND category <> 'Widget'")--(deftest get-transform-details-native-query-test- (mt/with-premium-features #{:transforms-basic :transforms-python :hosting}- (mt/with-current-user (mt/user->id :crowberto)- (mt/with-temp [:model/Transform {transform-id :id}- {:name "Gadget Products"- :source {:type "query"- :query (lib/native-query (mt/metadata-provider) gadget-sql)}}]- (let [{:keys [output]} (agent-transforms/get-transform-details-tool {:transform_id transform-id})]- (is (str/includes? output "name=\"Gadget Products\""))- (is (str/includes? output "<source type=\"query\">"))- (is (str/includes? output (str "<query>" gadget-sql "</query>"))))))))--(deftest get-transform-details-notebook-query-test- (mt/with-premium-features #{:transforms-basic :transforms-python :hosting}- (mt/with-current-user (mt/user->id :crowberto)- (mt/with-temp [:model/Transform {transform-id :id}- {:name "Notebook Products"- :source {:type "query"- :query (lib/query (mt/metadata-provider)- (lib.metadata/table (mt/metadata-provider) (mt/id :products)))}}]- (let [{:keys [output]} (agent-transforms/get-transform-details-tool {:transform_id transform-id})]- (is (str/includes? output "<query>\n```json"))- (is (str/includes? output "```\n</query>"))- (let [exported (json/decode (second (re-find #"(?s)```json\n(.*)\n```" output)))]- (is (= [(:name (lib.metadata/database (mt/metadata-provider))) "PUBLIC" "PRODUCTS"]- (get-in exported ["stages" 0 "source-table"])))- (is (not-any? #(and (map? %) (contains? % "lib/metadata"))- (tree-seq coll? seq exported)))))))))--(defn- query-transform-details!- "`get_transform_details` output for a transform whose source is `query`, read as rasta with query- access to the whole database - so `transforms/get-transform` passes and whatever happens to the- source is the export gate's doing. A data analyst's transforms access keys off- `advanced-permissions`, so that is enabled too - and since no OSS build can have it, every- caller is wrapped in [[mt/when-ee-evailable]]."- [query]- (mt/with-premium-features #{:transforms-basic :transforms-python :hosting :advanced-permissions}- (mt/with-temp [:model/Transform {transform-id :id}- {:name "Orders Rollup" :source {:type "query" :query query}}]- (mt/with-data-analyst-role! (mt/user->id :rasta)- (mt/with-current-user (mt/user->id :rasta)- (:output (agent-transforms/get-transform-details-tool {:transform_id transform-id})))))))--;; EE-only: the analyst reading here only can with `advanced-permissions`, which no OSS build can have-(mt/when-ee-evailable- (deftest get-transform-details-sandboxed-field-test- (testing "a source query naming a field the user's sandbox hides renders without its query, since- exporting it would resolve that field's id to a name"- (let [query (-> (lib/query (mt/metadata-provider)- (lib.metadata/table (mt/metadata-provider) (mt/id :venues)))- (lib/filter (lib/> (lib.metadata/field (mt/metadata-provider) (mt/id :venues :price)) 1)))]- (is (str/includes? (query-transform-details! query) "<query>"))- (mt/with-dynamic-fn-redefs [metabot.perms/sandbox-restricted-fields (fn [_table-ids] {(mt/id :venues) #{}})]- (is (not (str/includes? (query-transform-details! query) "<query>"))))))))--;; EE-only: the analyst reading here only can with `advanced-permissions`, which no OSS build can have-(mt/when-ee-evailable- (deftest get-transform-details-unpermissionable-source-test- (testing "a source query whose permissions cannot be calculated at all still renders, since that is- not a refusal - it just renders unresolved, naming nothing the user may not see"- (let [output (query-transform-details! {:database (mt/id)- :type :query- :query {:source-table "card__13371337"}})]- (is (str/includes? output "<query>"))- (is (str/includes? output ":source-card 13371337")- "the card id is still a number, so nothing was resolved to a name")))))--(defn- rendered-source- "The source query [[query-export/transform-with-exportable-source]] leaves on a transform whose- source is `query`, or nil when it withholds it."- [query]- (get-in (query-export/transform-with-exportable-source {:source {:type "query" :query query}})- [:source :query]))--(deftest transform-source-withholds-native-sql-behind-a-later-stage-test- (testing "a native stage under an MBQL stage is still native, so a check that could not be made- withholds it instead of printing the SQL in the unresolved fallback"- (let [native-stage {:lib/type :mbql.stage/native- :native "SELECT * FROM {{snip}}"- :template-tags {"snip" {:type :snippet- :name "snip"- :display-name "snip"- :snippet-id Integer/MAX_VALUE}}}]- (mt/with-current-user (mt/user->id :rasta)- (doseq [stages [[native-stage] [native-stage {:lib/type :mbql.stage/mbql}]]]- (is (nil? (rendered-source {:lib/type :mbql/query :database (mt/id) :stages stages}))- (str (count stages) " stage(s)")))))))--(deftest transform-source-survives-a-deleted-source-database-test- (testing "an orphaned transform keeps its source, which is what an admin repairs it from: there- is no metadata left behind a deleted database to resolve anything against"- (mt/with-temp [:model/Database {db-id :id} {}]- (t2/delete! :model/Database :id db-id)- (mt/with-current-user (mt/user->id :crowberto)- (is (str/includes? (str (rendered-source {:database db-id- :type :native- :native {:query "SELECT 1"}}))- "SELECT 1"))))))--;; EE-only: the analyst reading here only can with `advanced-permissions`, which no OSS build can have-(mt/when-ee-evailable- (deftest get-transform-details-unpermissionable-native-source-test- (testing "native SQL stays out when the check could not run, since its table and column names are- already plain text and having no provider hides nothing"- (let [output (query-transform-details!- {:database (mt/id)- :type :native- :native {:query "SELECT * FROM {{snip}}"- :template-tags {"snip" {:type :snippet- :name "snip"- :display-name "snip"- :snippet-id Integer/MAX_VALUE}}}})]- (is (not (str/includes? output "<query>")))- (is (not (str/includes? output "SELECT")))))))--;; EE-only: the analyst reading here only can with `advanced-permissions`, which no OSS build can have-(mt/when-ee-evailable- (deftest get-transform-details-source-permission-test- (testing "transforms/get-transform refuses a transform whose stored query the user cannot run, even- with query access to another table in its database, so the tool never reaches the source"- (mt/with-premium-features #{:transforms-basic :transforms-python :hosting :advanced-permissions}- (mt/with-temp [:model/Transform {transform-id :id}- {:name "Orders Rollup"- :source {:type "query"- :query (lib/query (mt/metadata-provider)- (lib.metadata/table (mt/metadata-provider) (mt/id :orders)))}}]- (mt/with-data-analyst-role! (mt/user->id :rasta)- (mt/with-no-data-perms-for-all-users!- (perms/set-table-permission! (perms-group/all-users) (mt/id :venues) :perms/view-data :unrestricted)- (perms/set-table-permission! (perms-group/all-users) (mt/id :venues) :perms/create-queries :query-builder)- (mt/with-current-user (mt/user->id :rasta)- (is (=? {:status-code 403 :output "You don't have permissions to do that."}- (agent-transforms/get-transform-details-tool {:transform_id transform-id}))))- (testing "and the query renders once the source table is granted"- (perms/set-table-permission! (perms-group/all-users) (mt/id :orders) :perms/view-data :unrestricted)- (perms/set-table-permission! (perms-group/all-users) (mt/id :orders) :perms/create-queries :query-builder)- (mt/with-current-user (mt/user->id :rasta)- (is (str/includes? (:output (agent-transforms/get-transform-details-tool {:transform_id transform-id}))- "<query>")))))))))))--(deftest get-transform-details-python-source-test- (mt/with-premium-features #{:transforms-basic :transforms-python :hosting}- (mt/with-current-user (mt/user->id :crowberto)- (mt/with-temp [:model/Transform {transform-id :id}- {:name "Gadget Metrics"- :source {:type "python"- :source-database (mt/id)- :body "import pandas as pd"}}]- (let [{:keys [output]} (agent-transforms/get-transform-details-tool {:transform_id transform-id})]- (is (str/includes? output "<source type=\"python\">"))- (is (str/includes? output "<body>import pandas as pd</body>"))- (is (str/includes? output (str "<database>" (mt/id) "</database>"))))))))--;; EE-only: the analyst reading here only can with `advanced-permissions`, which no OSS build can have-(mt/when-ee-evailable- (deftest get-transform-details-source-card-permission-test- ;; `advanced-permissions` is on so the refusal below is about the private source card, not the- ;; data analyst's missing entitlement- (mt/with-premium-features #{:transforms-basic :transforms-python :hosting :advanced-permissions}- (mt/with-non-admin-groups-no-root-collection-perms- (mt/with-full-data-perms-for-all-users!- (mt/with-data-analyst-role! (mt/user->id :rasta)- (mt/with-temp [:model/Collection {collection-id :id} {}- :model/Card {card-id :id}- {:collection_id collection-id- :database_id (mt/id)- :dataset_query (lib/query (mt/metadata-provider)- (lib.metadata/table (mt/metadata-provider) (mt/id :orders)))}- :model/Transform {transform-id :id}- {:name "Private source Card"- :source {:type "query"- :query (lib/query (mt/metadata-provider)- (lib.metadata/card (mt/metadata-provider) card-id))}}]- (mt/with-current-user (mt/user->id :rasta)- (let [{:keys [output status-code]} (agent-transforms/get-transform-details-tool {:transform_id transform-id})]- (is (= 403 status-code))- (is (= "You don't have permissions to do that." output)))))))))))