Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Convert Metabot search tools to the tool contract (#84947)

Andrei Simionescue1f019ae4cc4
diff --git a/src/metabase/metabot/tools/search.clj b/src/metabase/metabot/tools/search.clj
index b5abbdd92dc..82be2faf74e 100644
--- a/src/metabase/metabot/tools/search.clj
+++ b/src/metabase/metabot/tools/search.clj
@@ -10,6 +10,7 @@
[metabase.metabot.scope :as scope]
[metabase.metabot.search-models :as metabot.search-models]
[metabase.metabot.tmpl :as te]
+ [metabase.metabot.tools.core :as tools]
[metabase.metabot.tools.shared :as shared]
[metabase.metabot.tools.shared.instructions :as instructions]
[metabase.metabot.tools.shared.llm-shape :as llm-shape]
@@ -581,11 +582,6 @@
"<instructions>"
instructions/search-result-instructions "</instructions>")))
-(defn- invalid-entity-types
- [entity-types allowed]
- (when (seq entity-types)
- (seq (remove allowed entity-types))))
-
(def ^:private default-search-limit 10)
(def ^:private max-search-limit 50)
@@ -623,24 +619,20 @@
:collection (some-> (:collection r) (select-keys [:id :name])))))
(defn- do-search
- [label allowed-types search-opts {:keys [semantic_queries keyword_queries entity_types limit] :as _args}]
- (if-let [invalid (invalid-entity-types entity_types allowed-types)]
- {:output (str "Invalid entity_types for " label ": " (pr-str (vec invalid))
- ". Allowed types: " (str/join ", " allowed-types) ".")}
- (let [results (search (merge {:semantic-queries semantic_queries
- :term-queries keyword_queries
- :entity-types (or (seq entity_types) (vec allowed-types))
- :metabot shared/*metabot*
- :limit (min max-search-limit
- (or limit default-search-limit))}
- search-opts))]
- {:output (format-search-output results)
- :structured-output {:result-type :search
- :data results
- :total_count (count results)}
- :data-parts [(streaming/search-results-part
- {:total_count (count results)
- :results (mapv search-result->item results)})]})))
+ [default-types search-opts {:keys [semantic_queries keyword_queries entity_types limit]}]
+ (let [results (search (merge {:semantic-queries semantic_queries
+ :term-queries keyword_queries
+ :entity-types (or (seq entity_types) (vec default-types))
+ :limit (min max-search-limit
+ (or limit default-search-limit))}
+ search-opts))]
+ {:output (format-search-output results)
+ :structured-output {:result-type :search
+ :data results
+ :total_count (count results)}
+ :data-parts [(streaming/search-results-part
+ {:total_count (count results)
+ :results (mapv search-result->item results)})]}))
(def ^:private semantic-queries-schema
[:sequential {:error/message "must be an array of strings"}
@@ -689,13 +681,57 @@
;; ("Searching for …" while active, "Searched for …" once finished)
(str/join ", " queries))))
-(mu/defn ^{:tool-name "search"
- :scope scope/agent-search
- :title-fn search-display}
- search-tool
- "Find tables, models, metrics, measures, segments, dashboards, documents, and saved questions by topic across the instance. Use it when you don't know where something lives; once you have a hit, drill into it with read_resource rather than searching the same concept again."
- [args :- search-schema]
- (do-search "search" general-entity-types {} args))
+(tools/defrecoverable missing-query!
+ "A search needs at least one query and no blank entries."
+ {:payload [:map {:closed true}]}
+ [_]
+ {:message "Provide at least one query. Every query must be non-blank."
+ :recovery [{:uses #{"search"}
+ :text (str "Remove blank entries from every query list, then call `search` again "
+ "with non-blank queries in `keyword_queries`.")}]})
+
+(tools/defrecoverable unavailable-database!
+ "The database for this SQL search does not exist or the user cannot read it."
+ {:payload [:map {:closed true} [:database-id :int]]}
+ [{:keys [database-id]}]
+ {:message (str "Database " database-id
+ " was not found. It may not exist, or you may not have access to it.")
+ :recovery [{:uses #{}
+ :text "Use the database currently selected in the SQL editor."}
+ {:uses #{"read_resource"}
+ :text "Call `read_resource` with `metabase://databases` to find an accessible database ID."}]})
+
+(defn- check-search-database!
+ [database-id]
+ (when-not (and (pos-int? database-id)
+ (when-let [database (metabot.db/database-summary database-id)]
+ (mi/can-read? database)))
+ (unavailable-database! {:database-id database-id})))
+
+(defrecord SearchTool [description args default-types search-opts]
+ tools/Tool
+ (declaration [_]
+ {:name "search"
+ :description description
+ :args args
+ :scope scope/agent-search
+ :metabot/title-fn search-display})
+ (handle [_ {:keys [database_id semantic_queries keyword_queries] :as args} {:keys [metabot]}]
+ (let [queries (concat semantic_queries keyword_queries)]
+ (when (or (empty? queries) (some str/blank? queries))
+ (missing-query! {})))
+ (when (contains? args :database_id)
+ (check-search-database! database_id))
+ (do-search default-types
+ (cond-> (assoc search-opts :metabot metabot)
+ (contains? args :database_id) (assoc :database-id database_id))
+ args)))
+
+(def search-tool
+ "The general `search` tool."
+ (->SearchTool
+ "Find tables, models, metrics, measures, segments, dashboards, documents, and saved questions by topic across the instance. Use it when you don't know where something lives; once you have a hit, drill into it with read_resource rather than searching the same concept again."
+ search-schema general-entity-types {}))
(def ^:private sql-search-schema
[:map {:closed true}
@@ -705,13 +741,11 @@
[:entity_types {:optional true} (entity-types-schema "table" "model")]
[:limit {:optional true} limit-schema]])
-(mu/defn ^{:tool-name "search"
- :scope scope/agent-search
- :title-fn search-display}
- sql-search-tool
- "Find SQL-queryable data sources (tables and models) within a specific database by topic."
- [{:keys [database_id] :as args} :- sql-search-schema]
- (do-search "SQL search" (sorted-set "model" "table") {:database-id database_id} args))
+(def sql-search-tool
+ "The SQL `search` tool."
+ (->SearchTool
+ "Find SQL-queryable data sources (tables and models) within a specific database by topic."
+ sql-search-schema (sorted-set "model" "table") {}))
(def ^:private nlq-search-schema
[:map {:closed true}
@@ -720,16 +754,11 @@
[:entity_types {:optional true} (apply entity-types-schema general-entity-types)]
[:limit {:optional true} limit-schema]])
-(mu/defn ^{:tool-name "search"
- :scope scope/agent-search
- :title-fn search-display}
- nlq-search-tool
- "Find NLQ-queryable data sources by topic, or find dashboards and documents as save destinations."
- [{:keys [entity_types] :as args} :- nlq-search-schema]
- (let [args (cond-> args
- (not (seq entity_types))
- (assoc :entity_types ["metric" "measure" "segment" "model" "question" "table"]))]
- (do-search "NLQ search" general-entity-types {:profile-id "nlq"} args)))
+(def nlq-search-tool
+ "The NLQ `search` tool."
+ (->SearchTool
+ "Find NLQ-queryable data sources by topic, or find dashboards and documents as save destinations."
+ nlq-search-schema ["metric" "measure" "segment" "model" "question" "table"] {:profile-id "nlq"}))
(def ^:private transform-search-schema
[:map {:closed true}
@@ -745,5 +774,5 @@
transform-search-tool
"Find transforms, plus the tables and models around them, by topic."
[{:keys [search_native_query] :as args} :- transform-search-schema]
- (do-search "transform search" (sorted-set "model" "table" "transform")
- {:search-native-query search_native_query} args))
+ (do-search (sorted-set "model" "table" "transform")
+ {:search-native-query search_native_query :metabot shared/*metabot*} args))
diff --git a/test/metabase/metabot/agent/tool_output_test.clj b/test/metabase/metabot/agent/tool_output_test.clj
index abbf6968e77..5b5ac3a455f 100644
--- a/test/metabase/metabot/agent/tool_output_test.clj
+++ b/test/metabase/metabot/agent/tool_output_test.clj
@@ -198,10 +198,10 @@
(mt/test-driver :h2
(search/init-index! {:force-reset? false :re-populate? true})
(mt/with-current-user (mt/user->id :crowberto)
- (let [result (search-tools/search-tool
- {:semantic_queries ["orders"]
- :keyword_queries ["orders"]
- :entity_types ["table"]})]
+ (let [result (test-util/call-tool search-tools/search-tool
+ {:semantic_queries ["orders"]
+ :keyword_queries ["orders"]
+ :entity_types ["table"]})]
(assert-formatted-structured result "search: tables" #"<search-results\b"))))))
(deftest search-tool-with-models-structured-output-test
@@ -218,10 +218,10 @@
:name "Searchable Test Model"
:type :model}]
(search/init-index! {:force-reset? false :re-populate? true})
- (let [result (search-tools/search-tool
- {:semantic_queries ["Searchable Test Model"]
- :keyword_queries ["Searchable Test Model"]
- :entity_types ["model"]})]
+ (let [result (test-util/call-tool search-tools/search-tool
+ {:semantic_queries ["Searchable Test Model"]
+ :keyword_queries ["Searchable Test Model"]
+ :entity_types ["model"]})]
(assert-formatted-structured
result "search: model" #"<search-results\b"))))))))
diff --git a/test/metabase/metabot/tools/error_test.clj b/test/metabase/metabot/tools/error_test.clj
index 47c6199b306..3498c680fa3 100644
--- a/test/metabase/metabot/tools/error_test.clj
+++ b/test/metabase/metabot/tools/error_test.clj
@@ -3,6 +3,8 @@
[clojure.set :as set]
[clojure.test :refer :all]
[malli.generator :as mg]
+ [metabase.mcp.v2.api]
+ [metabase.mcp.v2.registry :as registry]
[metabase.metabot.agent.profiles :as profiles]
[metabase.metabot.tools.error :as tools.error]
[metabase.metabot.tools.legacy :as tools.legacy]
@@ -207,14 +209,16 @@
(is (some? (:output (tools.runtime/render error tool-names)))))))))))))
(def ^:private every-tool-name
- "Every model-facing tool name in the product, from the registered profiles.
+ "Every tool name a registered profile or MCP's registry declares.
- The universe a declaration's text is checked against, rather than one profile's: a message naming
- a tool is wrong wherever that tool is missing, and which profiles exist is not this test's
- business."
- (delay (into #{} (comp (mapcat :tools)
- (keep #(:name (tools.legacy/declaration-of %))))
- (vals @@#'profiles/*profiles))))
+ The universe a declaration's text and `:uses` are checked against, rather than one profile's: a
+ message naming a tool is wrong wherever that tool is missing, and which profiles exist is not this
+ test's business. MCP's tools are in it because one declaration can carry both surfaces' steps, as
+ `recoverable.pipeline-test` shows."
+ (delay (into (into #{} (comp (mapcat :tools)
+ (keep #(:name (tools.legacy/declaration-of %))))
+ (vals @@#'profiles/*profiles))
+ (keys @@#'registry/tools*))))
(defn- tools-named-in
"The tool names `text` names in backticks."
@@ -240,7 +244,13 @@
a profile that has no such tool"
(doseq [{:keys [uses text]} recovery]
(is (= #{} (set/difference (tools-named-in text) (set uses)))
- (str "add these to the :uses of the step " (pr-str text)))))))))
+ (str "add these to the :uses of the step " (pr-str text)))))
+ (testing "and a step may use only tools that exist, or renaming a tool would drop the step
+ from every profile"
+ (doseq [{:keys [uses text]} recovery]
+ (is (= #{} (set/difference (set uses) @every-tool-name))
+ (str "no tool has these names; update the :uses and the text of the step "
+ (pr-str text)))))))))
(deftest the-naming-check-finds-something-test
(testing "a predicate that stopped matching would make the test above pass vacuously"
diff --git a/test/metabase/metabot/tools/search_test.clj b/test/metabase/metabot/tools/search_test.clj
index 8dce91e5aca..0000110e8c7 100644
--- a/test/metabase/metabot/tools/search_test.clj
+++ b/test/metabase/metabot/tools/search_test.clj
@@ -7,6 +7,7 @@
[metabase.lib-be.metadata.jvm :as lib-be]
[metabase.lib.core :as lib]
[metabase.metabot.config :as metabot.config]
+ [metabase.metabot.scope :as scope]
[metabase.metabot.test-util :as test-util]
[metabase.metabot.tools :as metabot.tools]
[metabase.metabot.tools.search :as search]
@@ -21,6 +22,104 @@
[metabase.util :as u]
[toucan2.core :as t2]))
+(defn- invoke-search
+ ([tool-list args] (invoke-search tool-list args nil nil))
+ ([tool-list args metabot profile-id]
+ (binding [scope/*current-user-scope* #{scope/agent-search}]
+ ((get-in (metabot.tools/->entries tool-list (atom {}) metabot profile-id) ["search" :fn]) args))))
+
+(def ^:private missing-query-result
+ {:error {:class :recoverable :code ::search/missing-query}
+ :output (str "Provide at least one query. Every query must be non-blank.\n"
+ "Remove blank entries from every query list, then call `search` again "
+ "with non-blank queries in `keyword_queries`.")})
+
+(deftest ^:parallel search-requires-query-test
+ (mt/with-dynamic-fn-redefs [search/search (fn [_] (throw (ex-info "Search must not run" {})))]
+ (doseq [[tool-var extra-args] [[#'search/search-tool {}]
+ [#'search/sql-search-tool {:database_id 1}]
+ [#'search/nlq-search-tool {}]]
+ queries [{}
+ {:keyword_queries [] :semantic_queries []}
+ {:keyword_queries ["" " "] :semantic_queries ["\n"]}
+ {:keyword_queries ["" "orders"] :semantic_queries []}
+ {:keyword_queries ["orders" " "]}
+ {:keyword_queries ["orders"] :semantic_queries [""]}]]
+ (is (= missing-query-result
+ (invoke-search [tool-var] (merge extra-args queries)))))))
+
+(deftest sql-search-checks-database-test
+ (mt/with-no-data-perms-for-all-users!
+ (mt/with-test-user :rasta
+ (mt/with-dynamic-fn-redefs [search/search (fn [_] (throw (ex-info "Search must not run" {})))]
+ (doseq [database-id [Integer/MAX_VALUE (mt/id) 0 -1]]
+ (is (= {:error {:class :recoverable :code ::search/unavailable-database}
+ :output (str "Database " database-id
+ " was not found. It may not exist, or you may not have access to it.\n"
+ "Use the database currently selected in the SQL editor.\n"
+ "Call `read_resource` with `metabase://databases` to find an accessible database ID.")}
+ (invoke-search [#'search/sql-search-tool #'metabot.tools/read-resource-tool]
+ {:database_id database-id :keyword_queries ["orders"]}))))))))
+
+(deftest sql-search-non-admin-database-access-test
+ (search.tu/with-legacy-search
+ (let [args {:database_id (mt/id) :keyword_queries ["orders"] :entity_types ["table"]}
+ search-as-rasta (fn []
+ (mt/with-test-user :rasta
+ (is (false? api/*is-superuser?*))
+ (is (= (mt/user->id :rasta) api/*current-user-id*))
+ (invoke-search [#'search/sql-search-tool] args)))]
+ (testing "a non-admin with database access can search"
+ (let [result (search-as-rasta)]
+ (is (nil? (:error result)))
+ (is (= :search (get-in result [:structured-output :result-type])))
+ (is (some #(= (mt/id :orders) (:id %)) (get-in result [:structured-output :data])))))
+ (testing "the same non-admin is refused without database access"
+ (mt/with-no-data-perms-for-all-users!
+ (let [result (search-as-rasta)]
+ (is (= {:class :recoverable :code ::search/unavailable-database} (:error result)))
+ (is (= (str "Database " (mt/id)
+ " was not found. It may not exist, or you may not have access to it.\n"
+ "Use the database currently selected in the SQL editor.")
+ (:output result)))))))))
+
+(deftest ^:parallel sql-search-database-recovery-follows-profile-test
+ (mt/with-dynamic-fn-redefs [search/search (constantly [])]
+ (is (= {:error {:class :recoverable :code ::search/unavailable-database}
+ :output (str "Database " Integer/MAX_VALUE
+ " was not found. It may not exist, or you may not have access to it.\n"
+ "Use the database currently selected in the SQL editor.")}
+ (invoke-search [#'search/sql-search-tool]
+ {:database_id Integer/MAX_VALUE :keyword_queries ["orders"]})))))
+
+(deftest search-query-and-context-test
+ (mt/with-test-user :crowberto
+ (let [metabot {:entity_id "configured-search-bot" :collection_id 42 :use_verified_content true}]
+ (doseq [[tool-var extra-args search-opts]
+ [[#'search/search-tool {} {}]
+ [#'search/sql-search-tool {:database_id (mt/id)} {:database-id (mt/id)}]
+ [#'search/nlq-search-tool {} {:profile-id "nlq"}]]
+ queries [{:keyword_queries ["orders"]}
+ {:semantic_queries ["order revenue"]}]]
+ (let [captured (atom nil)]
+ (mt/with-dynamic-fn-redefs [search/search (fn [opts] (reset! captured opts) [])]
+ (is (=? {:structured-output {:result-type :search :data [] :total_count 0}
+ :data-parts [{:type :data :data-type "search_results"
+ :data {:results [] :total_count 0}}]}
+ (invoke-search [tool-var] (merge extra-args queries) metabot "internal")))
+ (is (=? (merge {:metabot metabot} search-opts) @captured))))))))
+
+(deftest ^:parallel unsupported-search-entity-types-test
+ (mt/with-dynamic-fn-redefs [search/search (fn [_] (throw (ex-info "Search must not run" {})))]
+ (doseq [[tool-var extra-args entity-type]
+ [[#'search/search-tool {} "database"]
+ [#'search/sql-search-tool {:database_id 1} "metric"]
+ [#'search/nlq-search-tool {} "transform"]]]
+ (let [result (invoke-search [tool-var] (merge extra-args {:keyword_queries ["orders"]
+ :entity_types [entity-type]}))]
+ (is (= {:class :validation :code :invalid-arguments} (:error result)))
+ (is (str/includes? (:output result) "entity_types"))))))
+
(deftest ^:parallel search-display-test
(testing "joins keyword and semantic queries into the search object (client owns the verb/tense)"
(is (= "revenue, orders"
@@ -458,7 +557,7 @@
(mt/with-dynamic-fn-redefs [search-core/ranked-results (fn [context]
(reset! captured (:models context))
[])]
- (search/nlq-search-tool {:keyword_queries ["x"]}))
+ (invoke-search [#'search/nlq-search-tool] {:keyword_queries ["x"]}))
(is (= #{"card"
"dataset"
"measure"
@@ -474,36 +573,36 @@
(mt/with-dynamic-fn-redefs [search-core/ranked-results (fn [context]
(reset! captured (:models context))
[])]
- (search/sql-search-tool {:keyword_queries ["x"] :database_id 1}))
+ (invoke-search [#'search/sql-search-tool] {:keyword_queries ["x"] :database_id (mt/id)}))
(is (= #{"table" "dataset"} @captured))))
(testing "general search includes documents in its default entity types"
(let [captured (atom nil)]
(mt/with-dynamic-fn-redefs [search-core/ranked-results (fn [context]
(reset! captured (:models context))
[])]
- (search/search-tool {:keyword_queries ["x"]}))
+ (invoke-search [#'search/search-tool] {:keyword_queries ["x"]}))
(is (contains? @captured "document"))))
(testing "agent-supplied entity_types narrow the default allowed set"
(let [captured (atom nil)]
(mt/with-dynamic-fn-redefs [search-core/ranked-results (fn [context]
(reset! captured (:models context))
[])]
- (search/nlq-search-tool {:keyword_queries ["x"] :entity_types ["metric"]}))
+ (invoke-search [#'search/nlq-search-tool] {:keyword_queries ["x"] :entity_types ["metric"]}))
(is (= #{"metric"} @captured))))
(testing "agent-supplied measure and segment types reach the engine unchanged"
(let [captured (atom nil)]
(mt/with-dynamic-fn-redefs [search-core/ranked-results (fn [context]
(reset! captured (:models context))
[])]
- (search/search-tool {:keyword_queries ["x"] :entity_types ["measure" "segment"]}))
+ (invoke-search [#'search/search-tool] {:keyword_queries ["x"] :entity_types ["measure" "segment"]}))
(is (= #{"measure" "segment"} @captured))))
(testing "NLQ search accepts document and dashboard destination types"
(let [captured (atom nil)]
(mt/with-dynamic-fn-redefs [search-core/ranked-results (fn [context]
(reset! captured (:models context))
[])]
- (search/nlq-search-tool {:keyword_queries ["plan"]
- :entity_types ["document" "dashboard"]}))
+ (invoke-search [#'search/nlq-search-tool] {:keyword_queries ["plan"]
+ :entity_types ["document" "dashboard"]}))
(is (= #{"document" "dashboard"} @captured))))))))
(deftest tool-limit-test
@@ -520,7 +619,7 @@
search-core/search-results (fn [context _ _]
(reset! captured (:limit-int context))
{:data [] :total 0})]
- (search/search-tool {:keyword_queries ["x"]}))
+ (invoke-search [#'search/search-tool] {:keyword_queries ["x"]}))
(is (= 10 @captured))))
(testing "explicit limit is honored"
(let [captured (atom nil)]
@@ -528,20 +627,22 @@
search-core/search-results (fn [context _ _]
(reset! captured (:limit-int context))
{:data [] :total 0})]
- (search/search-tool {:keyword_queries ["x"] :limit 25}))
+ (invoke-search [#'search/search-tool] {:keyword_queries ["x"] :limit 25}))
(is (= 25 @captured))))
- (testing "limit above 50 is rejected by schema validation"
- (is (thrown? Exception
- (search/search-tool {:keyword_queries ["x"] :limit 75}))))
- (testing "limit below 1 is rejected by schema validation"
- (is (thrown? Exception
- (search/search-tool {:keyword_queries ["x"] :limit 0}))))))))
+ (testing "limits below 1 or above 50 are rejected by schema validation"
+ (doseq [limit [0 75]]
+ (is (=? {:error {:class :validation :code :invalid-arguments}}
+ (invoke-search [#'search/search-tool] {:keyword_queries ["x"] :limit limit})))))))))
-(deftest ^:parallel search-failure-throws-test
- (testing "an exception from search propagates out of the tool (BOT-2199)"
+(deftest unexpected-search-failure-test
+ (mt/with-test-user :crowberto
(mt/with-dynamic-fn-redefs [search/search (fn [_] (throw (ex-info "Search index unavailable" {})))]
- (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Search index unavailable"
- (search/search-tool {:keyword_queries ["x"]}))))))
+ (doseq [[tool-var extra-args] [[#'search/search-tool {}]
+ [#'search/sql-search-tool {:database_id (mt/id)}]
+ [#'search/nlq-search-tool {}]]]
+ (is (= {:error {:class :unrecoverable :code :internal}
+ :output "This call failed and the user was shown the error (:internal). Don't retry it."}
+ (invoke-search [tool-var] (merge extra-args {:keyword_queries ["orders"]}))))))))
(deftest ^:parallel scalar-search-args-test
(testing "a scalar where an array is declared is rejected with guidance on how to repair the call"
diff --git a/test/metabase/metabot/tools_test.clj b/test/metabase/metabot/tools_test.clj
index a9ba6c1c399..a4f08f9359d 100644
--- a/test/metabase/metabot/tools_test.clj
+++ b/test/metabase/metabot/tools_test.clj
@@ -157,10 +157,10 @@
(is (some? args)))))))
(deftest search-tool-test
- (testing "search-tool var has valid metadata"
- (let [m (meta #'agent-tools/search-tool)]
- (is (= "search" (:tool-name m)))
- (is (some? (:schema m))))))
+ (testing "search declares its name and argument schema"
+ (let [declared (tools.legacy/declaration-of #'agent-tools/search-tool)]
+ (is (= "search" (:name declared)))
+ (is (some? (:args declared))))))
(deftest construct-notebook-query-tool-test
(testing "construct_notebook_query evaluates a representations query and creates a chart"