Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

BOT-1944: Fix two Metabot tool failures (#79498)

Andrei Simionescu660b7ac2db09
diff --git a/src/metabase/metabot/tools/shared/llm_shape.clj b/src/metabase/metabot/tools/shared/llm_shape.clj
index b454c325926..51aff97fe8f 100644
--- a/src/metabase/metabot/tools/shared/llm_shape.clj
+++ b/src/metabase/metabot/tools/shared/llm_shape.clj
@@ -18,6 +18,7 @@
[metabase.metabot.agent.prompts :as prompts]
[metabase.metabot.tmpl :as te]
[metabase.metabot.tools.shared.content-store :as shared.content-store]
+ [metabase.metabot.util :as metabot.u]
[metabase.util :as u]
[metabase.util.json :as json]
[metabase.util.log :as log]
@@ -88,8 +89,8 @@
(str/replace ">" ">")
(str/replace "\"" """))))
-(defn- escape-xml-content
- "Like `escape-xml` but leaves double-quotes intact. Safe for element/table-cell *content*
+(defn escape-xml-content
+ "Like [[escape-xml]] but leaves double-quotes intact. Safe for element/table-cell *content*
(only `& < >` are structural there), so a value such as a JSON portable-FK array renders
readably instead of with `&quot;` noise."
[s]
@@ -852,8 +853,7 @@
:transform_description description
:transform_source_type (some-> (:type source) clojure.core/name)
:transform_source_database (when-let [db (:source-database source)] (str db))
- :transform_source_query (let [q (:query source)]
- (when (string? q) q))
+ :transform_source_query (metabot.u/transform-query->text (:query source))
:transform_target (when target (pr-str target))}))
(def formatters
diff --git a/src/metabase/metabot/tools/sql.clj b/src/metabase/metabot/tools/sql.clj
index d6a1fc98101..9c3c58061f0 100644
--- a/src/metabase/metabot/tools/sql.clj
+++ b/src/metabase/metabot/tools/sql.clj
@@ -11,6 +11,7 @@
[metabase.metabot.tools.sql.create :as create-sql-query-tools]
[metabase.metabot.tools.sql.edit :as edit-sql-query-tools]
[metabase.metabot.tools.sql.replace :as replace-sql-query-tools]
+ [metabase.metabot.tools.util :as metabot.tools.u]
[metabase.util :as u]
[metabase.util.log :as log]
[metabase.util.malli :as mu]))
@@ -108,10 +109,7 @@
{:output (format-validation-error-output instr)
:instructions instr})))
(catch Exception e
- (log/errorf "Error creating SQL query: %s" (ex-message e))
- (if (:agent-error? (ex-data e))
- {:output (ex-message e)}
- {:output (str "Failed to create SQL query: " (or (ex-message e) "Unknown error"))}))))
+ (metabot.tools.u/handle-agent-error e))))
(mu/defn ^{:tool-name "create_sql_query"
:scope scope/agent-sql-create
@@ -119,25 +117,28 @@
create-sql-query-code-edit-tool
"Create a new SQL query and update the code editor buffer."
[{:keys [database_id sql_query]} :- create-sql-schema]
- (let [buffer-id (first-code-editor-buffer-id)]
- (if (nil? buffer-id)
- {:output "No active code editor buffer found for SQL editing."}
- (let [{:keys [validation-result action-result]}
- (create-sql-query-tools/create-sql-query
- {:database-id database_id
- :sql sql_query})
- {:keys [valid? dialect error-message]} validation-result
- {:keys [query-id query-content]} action-result]
- (if valid?
- (let [structured (assoc action-result :result-type :query)
- instr (instructions/query-created-instructions-for query-id)]
- {:output (format-query-output structured instr {:preamble? true})
- :structured-output structured
- :instructions instr
- :data-parts [(code-edit-part buffer-id query-content)]})
- (let [instr (instructions/sql-validation-error-instructions dialect error-message)]
- {:output (format-validation-error-output instr)
- :instructions instr}))))))
+ (try
+ (let [buffer-id (first-code-editor-buffer-id)]
+ (if (nil? buffer-id)
+ {:output "No active code editor buffer found for SQL editing."}
+ (let [{:keys [validation-result action-result]}
+ (create-sql-query-tools/create-sql-query
+ {:database-id database_id
+ :sql sql_query})
+ {:keys [valid? dialect error-message]} validation-result
+ {:keys [query-id query-content]} action-result]
+ (if valid?
+ (let [structured (assoc action-result :result-type :query)
+ instr (instructions/query-created-instructions-for query-id)]
+ {:output (format-query-output structured instr {:preamble? true})
+ :structured-output structured
+ :instructions instr
+ :data-parts [(code-edit-part buffer-id query-content)]})
+ (let [instr (instructions/sql-validation-error-instructions dialect error-message)]
+ {:output (format-validation-error-output instr)
+ :instructions instr})))))
+ (catch Exception e
+ (metabot.tools.u/handle-agent-error e))))
;;; ──────────────────────────────────────────────────────────────────
;;; Edit SQL query
diff --git a/src/metabase/metabot/tools/sql/create.clj b/src/metabase/metabot/tools/sql/create.clj
index 39402cca2a9..3f366d4a942 100644
--- a/src/metabase/metabot/tools/sql/create.clj
+++ b/src/metabase/metabot/tools/sql/create.clj
@@ -30,7 +30,12 @@
(throw (ex-info (tru "Database {0} not found" database-id)
{:agent-error? true
:database-id database-id})))
- (api/read-check :model/Database database-id))
+ (try
+ (api/read-check :model/Database database-id)
+ (catch clojure.lang.ExceptionInfo e
+ (throw (if (contains? #{403 404} (:status-code (ex-data e)))
+ (ex-info (ex-message e) (assoc (ex-data e) :agent-error? true) e)
+ e)))))
(mu/defn create-sql-query :- ::metabot.tools.sql.common/operation-result
"Create a new SQL query in memory.
diff --git a/src/metabase/metabot/tools/transforms.clj b/src/metabase/metabot/tools/transforms.clj
index 5f57ee3b258..638cb8c148d 100644
--- a/src/metabase/metabot/tools/transforms.clj
+++ b/src/metabase/metabot/tools/transforms.clj
@@ -3,9 +3,11 @@
SQL transform tools are implemented directly in OSS.
Python transform tools use defenterprise (return nil/error in OSS, real impl in EE)."
(:require
+ [clojure.string :as str]
[metabase.metabot.scope :as scope]
[metabase.metabot.tools.dependencies :as deps]
[metabase.metabot.tools.shared :as shared]
+ [metabase.metabot.tools.shared.llm-shape :as llm-shape]
[metabase.metabot.tools.transforms.write :as transforms-write-tools]
[metabase.metabot.tools.util :as metabot.tools.u]
[metabase.metabot.util :as metabot.u]
@@ -20,16 +22,30 @@
;;; Formatting helpers
;;; ──────────────────────────────────────────────────────────────────
+;; Hand-built rather than `metabot.u/xml` so query and body content stays verbatim:
+;; `clojure.data.xml` escapes `<`/`>`/`&`, and an escaped query breaks `old_string`
+;; matching when the model quotes it back to the write tools.
(defn- format-transform-details-output
- [{:keys [id name description source target] :as _transform}]
- (metabot.u/xml
- [:transform {:id id :name name}
- (when description [:description description])
- (when source
- [:source {:type (:type source)}
- (when (:query source) [:query (:query source)])
- (when (:source-database source) [:database (:source-database source)])])
- (when target [:target (pr-str target)])]))
+ [{:keys [id description source target] :as transform}]
+ (let [{source-type :type, :keys [query body source-database]} source]
+ (->> [(str "<transform id=\"" id "\" name=\"" (llm-shape/escape-xml (:name transform)) "\">")
+ (when description
+ (str " <description>" (llm-shape/escape-xml-content description) "</description>"))
+ (when source
+ (str " <source type=\"" (some-> source-type name) "\">"))
+ (when query
+ (str " <query>" (metabot.u/transform-query->text query) "</query>"))
+ (when body
+ (str " <body>" body "</body>"))
+ (when source-database
+ (str " <database>" source-database "</database>"))
+ (when source
+ " </source>")
+ (when target
+ (str " <target>" (llm-shape/escape-xml-content (pr-str target)) "</target>"))
+ "</transform>"]
+ (remove nil?)
+ (str/join "\n"))))
(defn format-transform-write-output
"Format the output of a transform write operation."
diff --git a/src/metabase/metabot/util.clj b/src/metabase/metabot/util.clj
index d4116a5197c..a5b8754b135 100644
--- a/src/metabase/metabot/util.clj
+++ b/src/metabase/metabot/util.clj
@@ -47,7 +47,25 @@
;; Following should be ideally handled by lib functions. However we have test in place that checks this piece
;; is able to handle not-normalized mbql5 with e.g. string value for type. Lib functions throw on such input.
;;
- ;; Try lib/query format (with stages)
- (get-in query [:stages 0 :native])
+ ;; Try lib/query format (with stages); stage 0 of a multi-stage query would be partial SQL, so fall through
+ (when (= 1 (count (:stages query)))
+ (get-in query [:stages 0 :native]))
;; Try legacy format
- (get-in query [:native :query])))
+ (get-in query [:native :query])
+ ;; orphaned sources skip normalization and keep their JSON string keys
+ (when (= 1 (count (get query "stages")))
+ (get-in query ["stages" 0 "native"]))
+ (get-in query ["native" "query"])))
+
+(defn transform-query->text
+ "Render a transform source query for model context: the native SQL when the query
+ has any, otherwise the EDN of the query with its metadata provider stripped so the
+ provider is never printed into the context window. String queries (legacy data and
+ orphaned sources, which skip normalization) pass through verbatim."
+ [query]
+ (when query
+ (cond
+ (string? query) query
+ (map? query) (or (extract-sql-content query)
+ (pr-str (dissoc query :lib/metadata)))
+ :else (pr-str query))))
diff --git a/test/metabase/metabot/tools/shared/llm_shape_test.clj b/test/metabase/metabot/tools/shared/llm_shape_test.clj
index 40cd3efd4a8..6b2eb1711d3 100644
--- a/test/metabase/metabot/tools/shared/llm_shape_test.clj
+++ b/test/metabase/metabot/tools/shared/llm_shape_test.clj
@@ -885,3 +885,23 @@
(testing "identity-only related tables render both the FK field name and id"
(is (str/includes? xml "related_by_field_name=\"user_id\" related_by_field_id=\"303\""))
(is (str/includes? xml "related_by_field_name=\"review_id\" related_by_field_id=\"404\""))))))
+
+(deftest ^:parallel transform->xml-source-query-test
+ (testing "a normalized (map) source query renders as verbatim SQL text"
+ (let [xml (llm-shape/transform->xml
+ {:id 7
+ :name "Orders rollup"
+ :source {:type :query
+ :query {:stages [{:lib/type :mbql.stage/native
+ :native "SELECT * FROM orders WHERE total < 100"}]}}})]
+ (is (str/includes? xml "<query>SELECT * FROM orders WHERE total < 100</query>"))))
+ (testing "a notebook-built source query renders as EDN with the metadata provider stripped"
+ (let [xml (llm-shape/transform->xml
+ {:id 8
+ :name "Notebook rollup"
+ :source {:type :query
+ :query {:lib/type :mbql/query
+ :lib/metadata :fake-metadata-provider
+ :stages [{:lib/type :mbql.stage/mbql :source-table 1}]}}})]
+ (is (str/includes? xml ":mbql.stage/mbql"))
+ (is (not (str/includes? xml ":lib/metadata"))))))
diff --git a/test/metabase/metabot/tools/sql_test.clj b/test/metabase/metabot/tools/sql_test.clj
index 6d2df4b4419..8419a747476 100644
--- a/test/metabase/metabot/tools/sql_test.clj
+++ b/test/metabase/metabot/tools/sql_test.clj
@@ -7,6 +7,7 @@
[metabase.lib.core :as lib]
[metabase.metabot.tools.shared :as shared]
[metabase.metabot.tools.sql :as agent-sql]
+ [metabase.metabot.tools.sql.create :as create-sql-query-tools]
[metabase.test :as mt]))
(deftest create-sql-query-output-test
@@ -46,6 +47,38 @@
(is (str/starts-with? (:instructions result) "The SQL query has a syntax error"))
(is (str/starts-with? (:output result) "<result>\nSQL query construction failed.\n</result>\n<instructions>\nThe SQL query has a syntax error"))))))))
+(defn- create-sql-query-in-code-editor
+ [args]
+ (binding [shared/*memory-atom* (atom {:context {:user_is_viewing [{:type "code_editor"
+ :buffers [{:id "buf-1"}]}]}})]
+ (agent-sql/create-sql-query-code-edit-tool (merge {:sql_query "SELECT 1"
+ :title "Results"}
+ args))))
+
+(deftest create-sql-query-code-edit-agent-error-output-test
+ (testing "create_sql_query in the code editor returns agent errors as output instead of throwing"
+ (mt/with-current-user (mt/user->id :crowberto)
+ (let [{:keys [output]} (create-sql-query-in-code-editor {:database_id Integer/MAX_VALUE})]
+ (is (str/includes? output "not found"))))))
+
+(deftest create-sql-query-code-edit-permission-error-output-test
+ (testing "create_sql_query in the code editor returns a permission failure as output with its status code"
+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}]
+ (mt/with-no-data-perms-for-all-users!
+ (mt/with-current-user (mt/user->id :rasta)
+ (let [{:keys [output status-code]} (create-sql-query-in-code-editor {:database_id db-id})]
+ (is (= 403 status-code))
+ (is (str/includes? output "permissions"))))))))
+
+(deftest create-sql-query-code-edit-unexpected-error-test
+ (testing "create_sql_query in the code editor rethrows non-agent errors so they stay tracked as failures"
+ (mt/with-current-user (mt/user->id :crowberto)
+ (mt/with-dynamic-fn-redefs [create-sql-query-tools/create-sql-query
+ (fn [& _] (throw (ex-info "boom" {})))]
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo #"boom"
+ (create-sql-query-in-code-editor {:database_id 1})))))))
+
(deftest edit-sql-query-output-test
(testing "edit_sql_query output includes edit-specific instructions with query ID"
(mt/test-drivers #{:h2}
diff --git a/test/metabase/metabot/tools/transforms_test.clj b/test/metabase/metabot/tools/transforms_test.clj
index 0b29ee5b9fa..679a82b5833 100644
--- a/test/metabase/metabot/tools/transforms_test.clj
+++ b/test/metabase/metabot/tools/transforms_test.clj
@@ -5,6 +5,7 @@
[clojure.string :as str]
[clojure.test :refer :all]
[metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
[metabase.metabot.tools.dependencies :as deps]
[metabase.metabot.tools.shared :as shared]
[metabase.metabot.tools.transforms :as agent-transforms]
@@ -12,6 +13,41 @@
[metabase.premium-features.core :as premium-features]
[metabase.test :as mt]))
+;;; ----------------------------------- read tool integration tests ---------------------------------------------------
+
+(deftest get-transform-details-tool-test
+ (mt/with-premium-features #{:transforms-basic :transforms-python :hosting}
+ (mt/with-current-user (mt/user->id :crowberto)
+ (testing "query source renders as verbatim SQL text, unescaped"
+ (mt/with-temp [:model/Transform {transform-id :id}
+ {:name "Gadget Products"
+ :source {:type "query"
+ :query (lib/native-query (mt/metadata-provider)
+ "SELECT * FROM products WHERE price < 100 AND category <> 'Widget'")}}]
+ (let [{:keys [output]} (agent-transforms/get-transform-details-tool {:transform_id transform-id})]
+ (is (str/includes? output "name=\"Gadget Products\""))
+ (is (str/includes? output "<source type=\"query\">"))
+ (is (str/includes? output "<query>SELECT * FROM products WHERE price < 100 AND category <> 'Widget'</query>")))))
+ (testing "notebook-built source falls back to EDN with the metadata provider stripped"
+ (mt/with-temp [:model/Transform {transform-id :id}
+ {:name "Notebook Products"
+ :source {:type "query"
+ :query (lib/query (mt/metadata-provider)
+ (lib.metadata/table (mt/metadata-provider) (mt/id :products)))}}]
+ (let [{:keys [output]} (agent-transforms/get-transform-details-tool {:transform_id transform-id})]
+ (is (str/includes? output ":lib/type"))
+ (is (not (str/includes? output ":lib/metadata"))))))
+ (testing "python source renders its body and source database"
+ (mt/with-temp [:model/Transform {transform-id :id}
+ {:name "Gadget Metrics"
+ :source {:type "python"
+ :source-database (mt/id)
+ :body "import pandas as pd"}}]
+ (let [{:keys [output]} (agent-transforms/get-transform-details-tool {:transform_id transform-id})]
+ (is (str/includes? output "<source type=\"python\">"))
+ (is (str/includes? output "<body>import pandas as pd</body>"))
+ (is (str/includes? output (str "<database>" (mt/id) "</database>")))))))))
+
;;; ----------------------------------- write tool integration tests --------------------------------------------------
(deftest write-transform-sql-tool-test
diff --git a/test/metabase/metabot/util_test.clj b/test/metabase/metabot/util_test.clj
new file mode 100644
index 00000000000..0dca1d009ba
--- /dev/null
+++ b/test/metabase/metabot/util_test.clj
@@ -0,0 +1,37 @@
+(ns metabase.metabot.util-test
+ (:require
+ [clojure.string :as str]
+ [clojure.test :refer :all]
+ [metabase.metabot.util :as metabot.u]))
+
+(deftest ^:parallel transform-query->text-test
+ (testing "native query renders as its SQL"
+ (is (= "SELECT 1"
+ (metabot.u/transform-query->text {:stages [{:lib/type :mbql.stage/native
+ :native "SELECT 1"}]}))))
+ (testing "legacy format renders as its SQL"
+ (is (= "SELECT 2"
+ (metabot.u/transform-query->text {:native {:query "SELECT 2"}}))))
+ (testing "notebook-built query renders as EDN without the metadata provider"
+ (let [text (metabot.u/transform-query->text {:lib/type :mbql/query
+ :lib/metadata :fake-provider
+ :stages [{:lib/type :mbql.stage/mbql}]})]
+ (is (str/includes? text ":mbql.stage/mbql"))
+ (is (not (str/includes? text ":lib/metadata")))))
+ (testing "raw string query passes through verbatim"
+ (is (= "SELECT * FROM legacy" (metabot.u/transform-query->text "SELECT * FROM legacy"))))
+ (testing "orphaned source (string keys, skipped normalization) renders as its SQL"
+ (is (= "SELECT 3"
+ (metabot.u/transform-query->text {"database" nil
+ "native" {"query" "SELECT 3"}})))
+ (is (= "SELECT 4"
+ (metabot.u/transform-query->text {"database" nil
+ "stages" [{"native" "SELECT 4"}]}))))
+ (testing "multi-stage query falls through to EDN rather than dropping stages"
+ (let [text (metabot.u/transform-query->text {:stages [{:lib/type :mbql.stage/native
+ :native "SELECT 5"}
+ {:lib/type :mbql.stage/mbql}]})]
+ (is (not= "SELECT 5" text))
+ (is (str/includes? text ":mbql.stage/mbql"))))
+ (testing "nil stays nil"
+ (is (nil? (metabot.u/transform-query->text nil)))))