The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
BOT-2540 A failing search ends Metabot's turn with "Something went wrong." (#85897)
Voytek Pituła4dff3b0df080
diff --git a/src/metabase/metabot/tools/search.clj b/src/metabase/metabot/tools/search.cljindex 8863d7964ec..c11df6d470c 100644--- a/src/metabase/metabot/tools/search.clj+++ b/src/metabase/metabot/tools/search.clj@@ -323,6 +323,14 @@result-lists (mapv deref futures)](reciprocal-rank-fusion result-lists))))+(defn- confined-collection-id+ "The collection that a search for `metabot` must stay inside, or nil. A confined metabot (embedded, or+ the nlq profile) may only search inside its own collection."+ [metabot profile-id]+ (when (or (= (:entity_id metabot) metabot.config/embedded-metabot-id)+ (= profile-id "nlq"))+ (:collection_id metabot)))+(defn search"Search for data sources (tables, models, cards, dashboards, metrics, measures, segments, transforms) in Metabase.Abstracted from the API endpoint logic.@@ -359,13 +367,10 @@metabot-search-models)_ (log/infof "[METABOT-SEARCH] Converted entity-types %s to search-models %s" entity-types search-models)use-verified? (:use_verified_content metabot)- embedded-metabot? (= (:entity_id metabot) metabot.config/embedded-metabot-id)- ;; A confined metabot (embedded, or the nlq profile) may only search inside its own- ;; collection. That is a containment boundary, not a default, so a caller-supplied+ ;; The confined collection is a containment boundary, not a default, so a caller-supplied;; collection-id — which the v2 search tool fills from a request filter — can never;; replace it. Unconfined, the caller's collection-id applies.- confined-id (when (or embedded-metabot? (= profile-id "nlq"))- (:collection_id metabot))+ confined-id (confined-collection-id metabot profile-id);; Measures and segments belong to a table, not a collection, so a search scoped to a collection;; never returns them, even though the search tools list them as entity types.collection-id (or confined-id collection-id)@@ -619,14 +624,65 @@:table_id (when (#{"measure" "segment"} (:type r)) (:base_table_id r)):collection (some-> (:collection r) (select-keys [:id :name])))))+(defn- browse-instead-step+ [& uri-text]+ {:uses #{"read_resource"}+ :text (apply str "Find the data with `read_resource` instead: " uri-text)})++(def ^:private no-retry-step+ {:uses #{"search"}+ :text (str "Do not call `search` again in this turn. If you cannot find the data another way, "+ "tell the user that search is unavailable right now.")})++(tools/defrecoverable search-failed!+ "The search backend failed, for example because the search index is missing.++ The browse step browses only the content that the failed search was limited to. `verified-only` is+ true when the search was limited to verified content but `read_resource` is not, as in the profiles+ exempt from curation; then there is no browse step. Every failure gets [[no-retry-step]], because+ each retry uses an iteration and logs a stack trace."+ {:payload [:map {:closed true}+ [:collection-id [:maybe :int]]+ [:database-id [:maybe :int]]+ [:verified-only :boolean]]}+ [{:keys [collection-id database-id verified-only]}]+ {:message "Search failed."+ :recovery (-> (cond+ verified-only+ []++ collection-id+ [(browse-instead-step "read `metabase://collection/" collection-id "/items`.")]++ database-id+ [(browse-instead-step "read `metabase://database/" database-id "/tables` and `metabase://database/"+ database-id "/models`.")]++ :else+ [(browse-instead-step "read `metabase://collections` for metrics, models, questions and "+ "dashboards, or `metabase://databases`, then `metabase://database/{id}/tables`, "+ "for tables.")])+ (conj no-retry-step))})+(defn- do-search[default-types search-opts {:keys [semantic_queries keyword_queries entity_types limit]}]- (let [results (search (merge {:semantic-queries semantic_queries- :term-queries keyword_queries- :entity-types (or (seq entity_types) (vec default-types))- :limit (min max-search-limit- (or limit default-search-limit))}- search-opts))]+ ;; Recoverable, not `:internal`: the agent can find the data without search, or tell the user that+ ;; search failed. `read_resource` does not use the search index.+ (let [opts (merge {:semantic-queries semantic_queries+ :term-queries keyword_queries+ :entity-types (or (seq entity_types) (vec default-types))+ :limit (min max-search-limit+ (or limit default-search-limit))}+ search-opts)+ results (try+ (search opts)+ (catch Exception e+ ;; The runtime logs a recoverable error at debug level, without its cause.+ (log/error e "Metabot search failed")+ (search-failed! {:collection-id (confined-collection-id (:metabot opts) (:profile-id opts))+ :database-id (:database-id opts)+ :verified-only (and (boolean (:use_verified_content (:metabot opts)))+ (not (shared/curated-only?)))})))]{:output (format-search-output results):structured-output {:result-type :search:data resultsdiff --git a/test/metabase/metabot/tools/search_test.clj b/test/metabase/metabot/tools/search_test.cljindex 0000110e8c7..e4fe5ef3208 100644--- a/test/metabase/metabot/tools/search_test.clj+++ b/test/metabase/metabot/tools/search_test.clj@@ -634,15 +634,46 @@(is (=? {:error {:class :validation :code :invalid-arguments}}(invoke-search [#'search/search-tool] {:keyword_queries ["x"] :limit limit})))))))))-(deftest unexpected-search-failure-test- (mt/with-test-user :crowberto- (mt/with-dynamic-fn-redefs [search/search (fn [_] (throw (ex-info "Search index unavailable" {})))]- (doseq [[tool-var extra-args] [[#'search/search-tool {}]- [#'search/sql-search-tool {:database_id (mt/id)}]- [#'search/nlq-search-tool {}]]]- (is (= {:error {:class :unrecoverable :code :internal}- :output "This call failed and the user was shown the error (:internal). Don't retry it."}- (invoke-search [tool-var] (merge extra-args {:keyword_queries ["orders"]}))))))))+(deftest ^:parallel search-failure-test+ (testing "a failing search backend lets the agent browse what the search was limited to (BOT-2540)"+ (mt/with-test-user :crowberto+ (mt/with-dynamic-fn-redefs [search/search (fn [_] (throw (ex-info "Search Index not found." {})))]+ (let [instance-step (str "Find the data with `read_resource` instead: read `metabase://collections` for "+ "metrics, models, questions and dashboards, or `metabase://databases`, then "+ "`metabase://database/{id}/tables`, for tables.")+ no-retry-step (str "Do not call `search` again in this turn. If you cannot find the data another "+ "way, tell the user that search is unavailable right now.")+ confined {:entity_id metabot.config/embedded-metabot-id :collection_id 42}]+ (doseq [[scope tool-var extra-args metabot profile-id step]+ [["the instance" #'search/search-tool {} nil "internal" instance-step]+ ["the instance" #'search/nlq-search-tool {} nil "nlq" instance-step]+ ["the instance" #'search/transform-search-tool {} nil "internal" instance-step]+ ["one database" #'search/sql-search-tool {:database_id (mt/id)} nil "sql"+ (str "Find the data with `read_resource` instead: read `metabase://database/" (mt/id)+ "/tables` and `metabase://database/" (mt/id) "/models`.")]+ ["a confined collection" #'search/search-tool {} confined "internal"+ "Find the data with `read_resource` instead: read `metabase://collection/42/items`."]+ ["a confined collection" #'search/nlq-search-tool {} {:collection_id 42} "nlq"+ "Find the data with `read_resource` instead: read `metabase://collection/42/items`."]+ ;; `read_resource` shows only verified content here too, so the browse step stays+ ["verified content" #'search/search-tool {} {:use_verified_content true} "internal" instance-step]+ ;; Profiles exempt from curation, where `read_resource` shows unverified content+ ["verified content" #'search/nlq-search-tool {} {:use_verified_content true} "nlq-fallback" nil]+ ["verified content" #'search/sql-search-tool {:database_id (mt/id)} {:use_verified_content true}+ "sql" nil]+ ["verified content" #'search/nlq-search-tool {} (assoc confined :use_verified_content true)+ "nlq" nil]]]+ (testing (str tool-var " limited to " scope " under " profile-id)+ (is (= {:error {:class :recoverable :code ::search/search-failed}+ :output (str/join "\n" (remove nil? ["Search failed." step no-retry-step]))}+ (invoke-search [tool-var #'metabot.tools/read-resource-tool]+ (merge extra-args {:keyword_queries ["orders"]})+ metabot+ profile-id)))))+ (testing "a profile without `read_resource`, like slackbot, is still told to stop searching"+ (is (= {:error {:class :recoverable :code ::search/search-failed}+ :output (str "Search failed.\n" no-retry-step)}+ (invoke-search [#'search/search-tool] {:keyword_queries ["orders"]})))))))))(deftest ^:parallel scalar-search-args-test(testing "a scalar where an array is declared is rejected with guidance on how to repair the call"