Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

BOT-2540 A failing search ends Metabot's turn with "Something went wrong." (#85897)

Voytek Pituła4dff3b0df080
diff --git a/src/metabase/metabot/tools/search.clj b/src/metabase/metabot/tools/search.clj
index 8863d7964ec..c11df6d470c 100644
--- a/src/metabase/metabot/tools/search.clj
+++ b/src/metabase/metabot/tools/search.clj
@@ -323,6 +323,14 @@
result-lists (mapv deref futures)]
(reciprocal-rank-fusion result-lists))))
+(defn- confined-collection-id
+ "The collection that a search for `metabot` must stay inside, or nil. A confined metabot (embedded, or
+ the nlq profile) may only search inside its own collection."
+ [metabot profile-id]
+ (when (or (= (:entity_id metabot) metabot.config/embedded-metabot-id)
+ (= profile-id "nlq"))
+ (:collection_id metabot)))
+
(defn search
"Search for data sources (tables, models, cards, dashboards, metrics, measures, segments, transforms) in Metabase.
Abstracted from the API endpoint logic.
@@ -359,13 +367,10 @@
metabot-search-models)
_ (log/infof "[METABOT-SEARCH] Converted entity-types %s to search-models %s" entity-types search-models)
use-verified? (:use_verified_content metabot)
- embedded-metabot? (= (:entity_id metabot) metabot.config/embedded-metabot-id)
- ;; A confined metabot (embedded, or the nlq profile) may only search inside its own
- ;; collection. That is a containment boundary, not a default, so a caller-supplied
+ ;; The confined collection is a containment boundary, not a default, so a caller-supplied
;; collection-id — which the v2 search tool fills from a request filter — can never
;; replace it. Unconfined, the caller's collection-id applies.
- confined-id (when (or embedded-metabot? (= profile-id "nlq"))
- (:collection_id metabot))
+ confined-id (confined-collection-id metabot profile-id)
;; Measures and segments belong to a table, not a collection, so a search scoped to a collection
;; never returns them, even though the search tools list them as entity types.
collection-id (or confined-id collection-id)
@@ -619,14 +624,65 @@
:table_id (when (#{"measure" "segment"} (:type r)) (:base_table_id r))
:collection (some-> (:collection r) (select-keys [:id :name])))))
+(defn- browse-instead-step
+ [& uri-text]
+ {:uses #{"read_resource"}
+ :text (apply str "Find the data with `read_resource` instead: " uri-text)})
+
+(def ^:private no-retry-step
+ {:uses #{"search"}
+ :text (str "Do not call `search` again in this turn. If you cannot find the data another way, "
+ "tell the user that search is unavailable right now.")})
+
+(tools/defrecoverable search-failed!
+ "The search backend failed, for example because the search index is missing.
+
+ The browse step browses only the content that the failed search was limited to. `verified-only` is
+ true when the search was limited to verified content but `read_resource` is not, as in the profiles
+ exempt from curation; then there is no browse step. Every failure gets [[no-retry-step]], because
+ each retry uses an iteration and logs a stack trace."
+ {:payload [:map {:closed true}
+ [:collection-id [:maybe :int]]
+ [:database-id [:maybe :int]]
+ [:verified-only :boolean]]}
+ [{:keys [collection-id database-id verified-only]}]
+ {:message "Search failed."
+ :recovery (-> (cond
+ verified-only
+ []
+
+ collection-id
+ [(browse-instead-step "read `metabase://collection/" collection-id "/items`.")]
+
+ database-id
+ [(browse-instead-step "read `metabase://database/" database-id "/tables` and `metabase://database/"
+ database-id "/models`.")]
+
+ :else
+ [(browse-instead-step "read `metabase://collections` for metrics, models, questions and "
+ "dashboards, or `metabase://databases`, then `metabase://database/{id}/tables`, "
+ "for tables.")])
+ (conj no-retry-step))})
+
(defn- do-search
[default-types search-opts {:keys [semantic_queries keyword_queries entity_types limit]}]
- (let [results (search (merge {:semantic-queries semantic_queries
- :term-queries keyword_queries
- :entity-types (or (seq entity_types) (vec default-types))
- :limit (min max-search-limit
- (or limit default-search-limit))}
- search-opts))]
+ ;; Recoverable, not `:internal`: the agent can find the data without search, or tell the user that
+ ;; search failed. `read_resource` does not use the search index.
+ (let [opts (merge {:semantic-queries semantic_queries
+ :term-queries keyword_queries
+ :entity-types (or (seq entity_types) (vec default-types))
+ :limit (min max-search-limit
+ (or limit default-search-limit))}
+ search-opts)
+ results (try
+ (search opts)
+ (catch Exception e
+ ;; The runtime logs a recoverable error at debug level, without its cause.
+ (log/error e "Metabot search failed")
+ (search-failed! {:collection-id (confined-collection-id (:metabot opts) (:profile-id opts))
+ :database-id (:database-id opts)
+ :verified-only (and (boolean (:use_verified_content (:metabot opts)))
+ (not (shared/curated-only?)))})))]
{:output (format-search-output results)
:structured-output {:result-type :search
:data results
diff --git a/test/metabase/metabot/tools/search_test.clj b/test/metabase/metabot/tools/search_test.clj
index 0000110e8c7..e4fe5ef3208 100644
--- a/test/metabase/metabot/tools/search_test.clj
+++ b/test/metabase/metabot/tools/search_test.clj
@@ -634,15 +634,46 @@
(is (=? {:error {:class :validation :code :invalid-arguments}}
(invoke-search [#'search/search-tool] {:keyword_queries ["x"] :limit limit})))))))))
-(deftest unexpected-search-failure-test
- (mt/with-test-user :crowberto
- (mt/with-dynamic-fn-redefs [search/search (fn [_] (throw (ex-info "Search index unavailable" {})))]
- (doseq [[tool-var extra-args] [[#'search/search-tool {}]
- [#'search/sql-search-tool {:database_id (mt/id)}]
- [#'search/nlq-search-tool {}]]]
- (is (= {:error {:class :unrecoverable :code :internal}
- :output "This call failed and the user was shown the error (:internal). Don't retry it."}
- (invoke-search [tool-var] (merge extra-args {:keyword_queries ["orders"]}))))))))
+(deftest ^:parallel search-failure-test
+ (testing "a failing search backend lets the agent browse what the search was limited to (BOT-2540)"
+ (mt/with-test-user :crowberto
+ (mt/with-dynamic-fn-redefs [search/search (fn [_] (throw (ex-info "Search Index not found." {})))]
+ (let [instance-step (str "Find the data with `read_resource` instead: read `metabase://collections` for "
+ "metrics, models, questions and dashboards, or `metabase://databases`, then "
+ "`metabase://database/{id}/tables`, for tables.")
+ no-retry-step (str "Do not call `search` again in this turn. If you cannot find the data another "
+ "way, tell the user that search is unavailable right now.")
+ confined {:entity_id metabot.config/embedded-metabot-id :collection_id 42}]
+ (doseq [[scope tool-var extra-args metabot profile-id step]
+ [["the instance" #'search/search-tool {} nil "internal" instance-step]
+ ["the instance" #'search/nlq-search-tool {} nil "nlq" instance-step]
+ ["the instance" #'search/transform-search-tool {} nil "internal" instance-step]
+ ["one database" #'search/sql-search-tool {:database_id (mt/id)} nil "sql"
+ (str "Find the data with `read_resource` instead: read `metabase://database/" (mt/id)
+ "/tables` and `metabase://database/" (mt/id) "/models`.")]
+ ["a confined collection" #'search/search-tool {} confined "internal"
+ "Find the data with `read_resource` instead: read `metabase://collection/42/items`."]
+ ["a confined collection" #'search/nlq-search-tool {} {:collection_id 42} "nlq"
+ "Find the data with `read_resource` instead: read `metabase://collection/42/items`."]
+ ;; `read_resource` shows only verified content here too, so the browse step stays
+ ["verified content" #'search/search-tool {} {:use_verified_content true} "internal" instance-step]
+ ;; Profiles exempt from curation, where `read_resource` shows unverified content
+ ["verified content" #'search/nlq-search-tool {} {:use_verified_content true} "nlq-fallback" nil]
+ ["verified content" #'search/sql-search-tool {:database_id (mt/id)} {:use_verified_content true}
+ "sql" nil]
+ ["verified content" #'search/nlq-search-tool {} (assoc confined :use_verified_content true)
+ "nlq" nil]]]
+ (testing (str tool-var " limited to " scope " under " profile-id)
+ (is (= {:error {:class :recoverable :code ::search/search-failed}
+ :output (str/join "\n" (remove nil? ["Search failed." step no-retry-step]))}
+ (invoke-search [tool-var #'metabot.tools/read-resource-tool]
+ (merge extra-args {:keyword_queries ["orders"]})
+ metabot
+ profile-id)))))
+ (testing "a profile without `read_resource`, like slackbot, is still told to stop searching"
+ (is (= {:error {:class :recoverable :code ::search/search-failed}
+ :output (str "Search failed.\n" no-retry-step)}
+ (invoke-search [#'search/search-tool] {:keyword_queries ["orders"]})))))))))
(deftest ^:parallel scalar-search-args-test
(testing "a scalar where an array is declared is rejected with guidance on how to repair the call"