The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Convert Metabot search tools to the tool contract (#84947)
Andrei Simionescue1f019ae4cc4
diff --git a/src/metabase/metabot/tools/search.clj b/src/metabase/metabot/tools/search.cljindex b5abbdd92dc..82be2faf74e 100644--- a/src/metabase/metabot/tools/search.clj+++ b/src/metabase/metabot/tools/search.clj@@ -10,6 +10,7 @@[metabase.metabot.scope :as scope][metabase.metabot.search-models :as metabot.search-models][metabase.metabot.tmpl :as te]+ [metabase.metabot.tools.core :as tools][metabase.metabot.tools.shared :as shared][metabase.metabot.tools.shared.instructions :as instructions][metabase.metabot.tools.shared.llm-shape :as llm-shape]@@ -581,11 +582,6 @@"<instructions>"instructions/search-result-instructions "</instructions>")))-(defn- invalid-entity-types- [entity-types allowed]- (when (seq entity-types)- (seq (remove allowed entity-types))))-(def ^:private default-search-limit 10)(def ^:private max-search-limit 50)@@ -623,24 +619,20 @@:collection (some-> (:collection r) (select-keys [:id :name])))))(defn- do-search- [label allowed-types search-opts {:keys [semantic_queries keyword_queries entity_types limit] :as _args}]- (if-let [invalid (invalid-entity-types entity_types allowed-types)]- {:output (str "Invalid entity_types for " label ": " (pr-str (vec invalid))- ". Allowed types: " (str/join ", " allowed-types) ".")}- (let [results (search (merge {:semantic-queries semantic_queries- :term-queries keyword_queries- :entity-types (or (seq entity_types) (vec allowed-types))- :metabot shared/*metabot*- :limit (min max-search-limit- (or limit default-search-limit))}- search-opts))]- {:output (format-search-output results)- :structured-output {:result-type :search- :data results- :total_count (count results)}- :data-parts [(streaming/search-results-part- {:total_count (count results)- :results (mapv search-result->item results)})]})))+ [default-types search-opts {:keys [semantic_queries keyword_queries entity_types limit]}]+ (let [results (search (merge {:semantic-queries semantic_queries+ :term-queries keyword_queries+ :entity-types (or (seq entity_types) (vec default-types))+ :limit (min max-search-limit+ (or limit default-search-limit))}+ search-opts))]+ {:output (format-search-output results)+ :structured-output {:result-type :search+ :data results+ :total_count (count results)}+ :data-parts [(streaming/search-results-part+ {:total_count (count results)+ :results (mapv search-result->item results)})]}))(def ^:private semantic-queries-schema[:sequential {:error/message "must be an array of strings"}@@ -689,13 +681,57 @@;; ("Searching for …" while active, "Searched for …" once finished)(str/join ", " queries))))-(mu/defn ^{:tool-name "search"- :scope scope/agent-search- :title-fn search-display}- search-tool- "Find tables, models, metrics, measures, segments, dashboards, documents, and saved questions by topic across the instance. Use it when you don't know where something lives; once you have a hit, drill into it with read_resource rather than searching the same concept again."- [args :- search-schema]- (do-search "search" general-entity-types {} args))+(tools/defrecoverable missing-query!+ "A search needs at least one query and no blank entries."+ {:payload [:map {:closed true}]}+ [_]+ {:message "Provide at least one query. Every query must be non-blank."+ :recovery [{:uses #{"search"}+ :text (str "Remove blank entries from every query list, then call `search` again "+ "with non-blank queries in `keyword_queries`.")}]})++(tools/defrecoverable unavailable-database!+ "The database for this SQL search does not exist or the user cannot read it."+ {:payload [:map {:closed true} [:database-id :int]]}+ [{:keys [database-id]}]+ {:message (str "Database " database-id+ " was not found. It may not exist, or you may not have access to it.")+ :recovery [{:uses #{}+ :text "Use the database currently selected in the SQL editor."}+ {:uses #{"read_resource"}+ :text "Call `read_resource` with `metabase://databases` to find an accessible database ID."}]})++(defn- check-search-database!+ [database-id]+ (when-not (and (pos-int? database-id)+ (when-let [database (metabot.db/database-summary database-id)]+ (mi/can-read? database)))+ (unavailable-database! {:database-id database-id})))++(defrecord SearchTool [description args default-types search-opts]+ tools/Tool+ (declaration [_]+ {:name "search"+ :description description+ :args args+ :scope scope/agent-search+ :metabot/title-fn search-display})+ (handle [_ {:keys [database_id semantic_queries keyword_queries] :as args} {:keys [metabot]}]+ (let [queries (concat semantic_queries keyword_queries)]+ (when (or (empty? queries) (some str/blank? queries))+ (missing-query! {})))+ (when (contains? args :database_id)+ (check-search-database! database_id))+ (do-search default-types+ (cond-> (assoc search-opts :metabot metabot)+ (contains? args :database_id) (assoc :database-id database_id))+ args)))++(def search-tool+ "The general `search` tool."+ (->SearchTool+ "Find tables, models, metrics, measures, segments, dashboards, documents, and saved questions by topic across the instance. Use it when you don't know where something lives; once you have a hit, drill into it with read_resource rather than searching the same concept again."+ search-schema general-entity-types {}))(def ^:private sql-search-schema[:map {:closed true}@@ -705,13 +741,11 @@[:entity_types {:optional true} (entity-types-schema "table" "model")][:limit {:optional true} limit-schema]])-(mu/defn ^{:tool-name "search"- :scope scope/agent-search- :title-fn search-display}- sql-search-tool- "Find SQL-queryable data sources (tables and models) within a specific database by topic."- [{:keys [database_id] :as args} :- sql-search-schema]- (do-search "SQL search" (sorted-set "model" "table") {:database-id database_id} args))+(def sql-search-tool+ "The SQL `search` tool."+ (->SearchTool+ "Find SQL-queryable data sources (tables and models) within a specific database by topic."+ sql-search-schema (sorted-set "model" "table") {}))(def ^:private nlq-search-schema[:map {:closed true}@@ -720,16 +754,11 @@[:entity_types {:optional true} (apply entity-types-schema general-entity-types)][:limit {:optional true} limit-schema]])-(mu/defn ^{:tool-name "search"- :scope scope/agent-search- :title-fn search-display}- nlq-search-tool- "Find NLQ-queryable data sources by topic, or find dashboards and documents as save destinations."- [{:keys [entity_types] :as args} :- nlq-search-schema]- (let [args (cond-> args- (not (seq entity_types))- (assoc :entity_types ["metric" "measure" "segment" "model" "question" "table"]))]- (do-search "NLQ search" general-entity-types {:profile-id "nlq"} args)))+(def nlq-search-tool+ "The NLQ `search` tool."+ (->SearchTool+ "Find NLQ-queryable data sources by topic, or find dashboards and documents as save destinations."+ nlq-search-schema ["metric" "measure" "segment" "model" "question" "table"] {:profile-id "nlq"}))(def ^:private transform-search-schema[:map {:closed true}@@ -745,5 +774,5 @@transform-search-tool"Find transforms, plus the tables and models around them, by topic."[{:keys [search_native_query] :as args} :- transform-search-schema]- (do-search "transform search" (sorted-set "model" "table" "transform")- {:search-native-query search_native_query} args))+ (do-search (sorted-set "model" "table" "transform")+ {:search-native-query search_native_query :metabot shared/*metabot*} args))diff --git a/test/metabase/metabot/agent/tool_output_test.clj b/test/metabase/metabot/agent/tool_output_test.cljindex abbf6968e77..5b5ac3a455f 100644--- a/test/metabase/metabot/agent/tool_output_test.clj+++ b/test/metabase/metabot/agent/tool_output_test.clj@@ -198,10 +198,10 @@(mt/test-driver :h2(search/init-index! {:force-reset? false :re-populate? true})(mt/with-current-user (mt/user->id :crowberto)- (let [result (search-tools/search-tool- {:semantic_queries ["orders"]- :keyword_queries ["orders"]- :entity_types ["table"]})]+ (let [result (test-util/call-tool search-tools/search-tool+ {:semantic_queries ["orders"]+ :keyword_queries ["orders"]+ :entity_types ["table"]})](assert-formatted-structured result "search: tables" #"<search-results\b"))))))(deftest search-tool-with-models-structured-output-test@@ -218,10 +218,10 @@:name "Searchable Test Model":type :model}](search/init-index! {:force-reset? false :re-populate? true})- (let [result (search-tools/search-tool- {:semantic_queries ["Searchable Test Model"]- :keyword_queries ["Searchable Test Model"]- :entity_types ["model"]})]+ (let [result (test-util/call-tool search-tools/search-tool+ {:semantic_queries ["Searchable Test Model"]+ :keyword_queries ["Searchable Test Model"]+ :entity_types ["model"]})](assert-formatted-structuredresult "search: model" #"<search-results\b"))))))))diff --git a/test/metabase/metabot/tools/error_test.clj b/test/metabase/metabot/tools/error_test.cljindex 47c6199b306..3498c680fa3 100644--- a/test/metabase/metabot/tools/error_test.clj+++ b/test/metabase/metabot/tools/error_test.clj@@ -3,6 +3,8 @@[clojure.set :as set][clojure.test :refer :all][malli.generator :as mg]+ [metabase.mcp.v2.api]+ [metabase.mcp.v2.registry :as registry][metabase.metabot.agent.profiles :as profiles][metabase.metabot.tools.error :as tools.error][metabase.metabot.tools.legacy :as tools.legacy]@@ -207,14 +209,16 @@(is (some? (:output (tools.runtime/render error tool-names)))))))))))))(def ^:private every-tool-name- "Every model-facing tool name in the product, from the registered profiles.+ "Every tool name a registered profile or MCP's registry declares.- The universe a declaration's text is checked against, rather than one profile's: a message naming- a tool is wrong wherever that tool is missing, and which profiles exist is not this test's- business."- (delay (into #{} (comp (mapcat :tools)- (keep #(:name (tools.legacy/declaration-of %))))- (vals @@#'profiles/*profiles))))+ The universe a declaration's text and `:uses` are checked against, rather than one profile's: a+ message naming a tool is wrong wherever that tool is missing, and which profiles exist is not this+ test's business. MCP's tools are in it because one declaration can carry both surfaces' steps, as+ `recoverable.pipeline-test` shows."+ (delay (into (into #{} (comp (mapcat :tools)+ (keep #(:name (tools.legacy/declaration-of %))))+ (vals @@#'profiles/*profiles))+ (keys @@#'registry/tools*))))(defn- tools-named-in"The tool names `text` names in backticks."@@ -240,7 +244,13 @@a profile that has no such tool"(doseq [{:keys [uses text]} recovery](is (= #{} (set/difference (tools-named-in text) (set uses)))- (str "add these to the :uses of the step " (pr-str text)))))))))+ (str "add these to the :uses of the step " (pr-str text)))))+ (testing "and a step may use only tools that exist, or renaming a tool would drop the step+ from every profile"+ (doseq [{:keys [uses text]} recovery]+ (is (= #{} (set/difference (set uses) @every-tool-name))+ (str "no tool has these names; update the :uses and the text of the step "+ (pr-str text)))))))))(deftest the-naming-check-finds-something-test(testing "a predicate that stopped matching would make the test above pass vacuously"diff --git a/test/metabase/metabot/tools/search_test.clj b/test/metabase/metabot/tools/search_test.cljindex 8dce91e5aca..0000110e8c7 100644--- a/test/metabase/metabot/tools/search_test.clj+++ b/test/metabase/metabot/tools/search_test.clj@@ -7,6 +7,7 @@[metabase.lib-be.metadata.jvm :as lib-be][metabase.lib.core :as lib][metabase.metabot.config :as metabot.config]+ [metabase.metabot.scope :as scope][metabase.metabot.test-util :as test-util][metabase.metabot.tools :as metabot.tools][metabase.metabot.tools.search :as search]@@ -21,6 +22,104 @@[metabase.util :as u][toucan2.core :as t2]))+(defn- invoke-search+ ([tool-list args] (invoke-search tool-list args nil nil))+ ([tool-list args metabot profile-id]+ (binding [scope/*current-user-scope* #{scope/agent-search}]+ ((get-in (metabot.tools/->entries tool-list (atom {}) metabot profile-id) ["search" :fn]) args))))++(def ^:private missing-query-result+ {:error {:class :recoverable :code ::search/missing-query}+ :output (str "Provide at least one query. Every query must be non-blank.\n"+ "Remove blank entries from every query list, then call `search` again "+ "with non-blank queries in `keyword_queries`.")})++(deftest ^:parallel search-requires-query-test+ (mt/with-dynamic-fn-redefs [search/search (fn [_] (throw (ex-info "Search must not run" {})))]+ (doseq [[tool-var extra-args] [[#'search/search-tool {}]+ [#'search/sql-search-tool {:database_id 1}]+ [#'search/nlq-search-tool {}]]+ queries [{}+ {:keyword_queries [] :semantic_queries []}+ {:keyword_queries ["" " "] :semantic_queries ["\n"]}+ {:keyword_queries ["" "orders"] :semantic_queries []}+ {:keyword_queries ["orders" " "]}+ {:keyword_queries ["orders"] :semantic_queries [""]}]]+ (is (= missing-query-result+ (invoke-search [tool-var] (merge extra-args queries)))))))++(deftest sql-search-checks-database-test+ (mt/with-no-data-perms-for-all-users!+ (mt/with-test-user :rasta+ (mt/with-dynamic-fn-redefs [search/search (fn [_] (throw (ex-info "Search must not run" {})))]+ (doseq [database-id [Integer/MAX_VALUE (mt/id) 0 -1]]+ (is (= {:error {:class :recoverable :code ::search/unavailable-database}+ :output (str "Database " database-id+ " was not found. It may not exist, or you may not have access to it.\n"+ "Use the database currently selected in the SQL editor.\n"+ "Call `read_resource` with `metabase://databases` to find an accessible database ID.")}+ (invoke-search [#'search/sql-search-tool #'metabot.tools/read-resource-tool]+ {:database_id database-id :keyword_queries ["orders"]}))))))))++(deftest sql-search-non-admin-database-access-test+ (search.tu/with-legacy-search+ (let [args {:database_id (mt/id) :keyword_queries ["orders"] :entity_types ["table"]}+ search-as-rasta (fn []+ (mt/with-test-user :rasta+ (is (false? api/*is-superuser?*))+ (is (= (mt/user->id :rasta) api/*current-user-id*))+ (invoke-search [#'search/sql-search-tool] args)))]+ (testing "a non-admin with database access can search"+ (let [result (search-as-rasta)]+ (is (nil? (:error result)))+ (is (= :search (get-in result [:structured-output :result-type])))+ (is (some #(= (mt/id :orders) (:id %)) (get-in result [:structured-output :data])))))+ (testing "the same non-admin is refused without database access"+ (mt/with-no-data-perms-for-all-users!+ (let [result (search-as-rasta)]+ (is (= {:class :recoverable :code ::search/unavailable-database} (:error result)))+ (is (= (str "Database " (mt/id)+ " was not found. It may not exist, or you may not have access to it.\n"+ "Use the database currently selected in the SQL editor.")+ (:output result)))))))))++(deftest ^:parallel sql-search-database-recovery-follows-profile-test+ (mt/with-dynamic-fn-redefs [search/search (constantly [])]+ (is (= {:error {:class :recoverable :code ::search/unavailable-database}+ :output (str "Database " Integer/MAX_VALUE+ " was not found. It may not exist, or you may not have access to it.\n"+ "Use the database currently selected in the SQL editor.")}+ (invoke-search [#'search/sql-search-tool]+ {:database_id Integer/MAX_VALUE :keyword_queries ["orders"]})))))++(deftest search-query-and-context-test+ (mt/with-test-user :crowberto+ (let [metabot {:entity_id "configured-search-bot" :collection_id 42 :use_verified_content true}]+ (doseq [[tool-var extra-args search-opts]+ [[#'search/search-tool {} {}]+ [#'search/sql-search-tool {:database_id (mt/id)} {:database-id (mt/id)}]+ [#'search/nlq-search-tool {} {:profile-id "nlq"}]]+ queries [{:keyword_queries ["orders"]}+ {:semantic_queries ["order revenue"]}]]+ (let [captured (atom nil)]+ (mt/with-dynamic-fn-redefs [search/search (fn [opts] (reset! captured opts) [])]+ (is (=? {:structured-output {:result-type :search :data [] :total_count 0}+ :data-parts [{:type :data :data-type "search_results"+ :data {:results [] :total_count 0}}]}+ (invoke-search [tool-var] (merge extra-args queries) metabot "internal")))+ (is (=? (merge {:metabot metabot} search-opts) @captured))))))))++(deftest ^:parallel unsupported-search-entity-types-test+ (mt/with-dynamic-fn-redefs [search/search (fn [_] (throw (ex-info "Search must not run" {})))]+ (doseq [[tool-var extra-args entity-type]+ [[#'search/search-tool {} "database"]+ [#'search/sql-search-tool {:database_id 1} "metric"]+ [#'search/nlq-search-tool {} "transform"]]]+ (let [result (invoke-search [tool-var] (merge extra-args {:keyword_queries ["orders"]+ :entity_types [entity-type]}))]+ (is (= {:class :validation :code :invalid-arguments} (:error result)))+ (is (str/includes? (:output result) "entity_types"))))))+(deftest ^:parallel search-display-test(testing "joins keyword and semantic queries into the search object (client owns the verb/tense)"(is (= "revenue, orders"@@ -458,7 +557,7 @@(mt/with-dynamic-fn-redefs [search-core/ranked-results (fn [context](reset! captured (:models context))[])]- (search/nlq-search-tool {:keyword_queries ["x"]}))+ (invoke-search [#'search/nlq-search-tool] {:keyword_queries ["x"]}))(is (= #{"card""dataset""measure"@@ -474,36 +573,36 @@(mt/with-dynamic-fn-redefs [search-core/ranked-results (fn [context](reset! captured (:models context))[])]- (search/sql-search-tool {:keyword_queries ["x"] :database_id 1}))+ (invoke-search [#'search/sql-search-tool] {:keyword_queries ["x"] :database_id (mt/id)}))(is (= #{"table" "dataset"} @captured))))(testing "general search includes documents in its default entity types"(let [captured (atom nil)](mt/with-dynamic-fn-redefs [search-core/ranked-results (fn [context](reset! captured (:models context))[])]- (search/search-tool {:keyword_queries ["x"]}))+ (invoke-search [#'search/search-tool] {:keyword_queries ["x"]}))(is (contains? @captured "document"))))(testing "agent-supplied entity_types narrow the default allowed set"(let [captured (atom nil)](mt/with-dynamic-fn-redefs [search-core/ranked-results (fn [context](reset! captured (:models context))[])]- (search/nlq-search-tool {:keyword_queries ["x"] :entity_types ["metric"]}))+ (invoke-search [#'search/nlq-search-tool] {:keyword_queries ["x"] :entity_types ["metric"]}))(is (= #{"metric"} @captured))))(testing "agent-supplied measure and segment types reach the engine unchanged"(let [captured (atom nil)](mt/with-dynamic-fn-redefs [search-core/ranked-results (fn [context](reset! captured (:models context))[])]- (search/search-tool {:keyword_queries ["x"] :entity_types ["measure" "segment"]}))+ (invoke-search [#'search/search-tool] {:keyword_queries ["x"] :entity_types ["measure" "segment"]}))(is (= #{"measure" "segment"} @captured))))(testing "NLQ search accepts document and dashboard destination types"(let [captured (atom nil)](mt/with-dynamic-fn-redefs [search-core/ranked-results (fn [context](reset! captured (:models context))[])]- (search/nlq-search-tool {:keyword_queries ["plan"]- :entity_types ["document" "dashboard"]}))+ (invoke-search [#'search/nlq-search-tool] {:keyword_queries ["plan"]+ :entity_types ["document" "dashboard"]}))(is (= #{"document" "dashboard"} @captured))))))))(deftest tool-limit-test@@ -520,7 +619,7 @@search-core/search-results (fn [context _ _](reset! captured (:limit-int context)){:data [] :total 0})]- (search/search-tool {:keyword_queries ["x"]}))+ (invoke-search [#'search/search-tool] {:keyword_queries ["x"]}))(is (= 10 @captured))))(testing "explicit limit is honored"(let [captured (atom nil)]@@ -528,20 +627,22 @@search-core/search-results (fn [context _ _](reset! captured (:limit-int context)){:data [] :total 0})]- (search/search-tool {:keyword_queries ["x"] :limit 25}))+ (invoke-search [#'search/search-tool] {:keyword_queries ["x"] :limit 25}))(is (= 25 @captured))))- (testing "limit above 50 is rejected by schema validation"- (is (thrown? Exception- (search/search-tool {:keyword_queries ["x"] :limit 75}))))- (testing "limit below 1 is rejected by schema validation"- (is (thrown? Exception- (search/search-tool {:keyword_queries ["x"] :limit 0}))))))))+ (testing "limits below 1 or above 50 are rejected by schema validation"+ (doseq [limit [0 75]]+ (is (=? {:error {:class :validation :code :invalid-arguments}}+ (invoke-search [#'search/search-tool] {:keyword_queries ["x"] :limit limit})))))))))-(deftest ^:parallel search-failure-throws-test- (testing "an exception from search propagates out of the tool (BOT-2199)"+(deftest unexpected-search-failure-test+ (mt/with-test-user :crowberto(mt/with-dynamic-fn-redefs [search/search (fn [_] (throw (ex-info "Search index unavailable" {})))]- (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Search index unavailable"- (search/search-tool {:keyword_queries ["x"]}))))))+ (doseq [[tool-var extra-args] [[#'search/search-tool {}]+ [#'search/sql-search-tool {:database_id (mt/id)}]+ [#'search/nlq-search-tool {}]]]+ (is (= {:error {:class :unrecoverable :code :internal}+ :output "This call failed and the user was shown the error (:internal). Don't retry it."}+ (invoke-search [tool-var] (merge extra-args {:keyword_queries ["orders"]}))))))))(deftest ^:parallel scalar-search-args-test(testing "a scalar where an array is declared is rejected with guidance on how to repair the call"diff --git a/test/metabase/metabot/tools_test.clj b/test/metabase/metabot/tools_test.cljindex a9ba6c1c399..a4f08f9359d 100644--- a/test/metabase/metabot/tools_test.clj+++ b/test/metabase/metabot/tools_test.clj@@ -157,10 +157,10 @@(is (some? args)))))))(deftest search-tool-test- (testing "search-tool var has valid metadata"- (let [m (meta #'agent-tools/search-tool)]- (is (= "search" (:tool-name m)))- (is (some? (:schema m))))))+ (testing "search declares its name and argument schema"+ (let [declared (tools.legacy/declaration-of #'agent-tools/search-tool)]+ (is (= "search" (:name declared)))+ (is (some? (:args declared))))))(deftest construct-notebook-query-tool-test(testing "construct_notebook_query evaluates a representations query and creates a chart"