Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Keep partial Metabot search results and explain empty ones (#85174)

Andrei Simionescub76292cc81b1
diff --git a/src/metabase/metabot/tools/search.clj b/src/metabase/metabot/tools/search.clj
index 1e225690251..8c1bc36940a 100644
--- a/src/metabase/metabot/tools/search.clj
+++ b/src/metabase/metabot/tools/search.clj
@@ -310,18 +310,20 @@
(sort-by :rrf >)
(map :search-result)))))))
-(defn- join-results-by-rrf
- "Execute multiple search queries in parallel and combine results using Reciprocal Rank Fusion.
- Items appearing in multiple result lists are boosted in the final ranking.
- May return more results than requested limit."
+(defn- run-searches
+ "Execute multiple search queries in parallel. Returns one `{:ranked results}` per query, or `{:error e}` for a
+ query whose search threw, so one failed query doesn't discard the others' results."
[search-fn search-engine all-queries]
- ;; Zero queries case is handled nicely by the >1 branch
- (if (= 1 (count all-queries))
- (search-fn (first all-queries) search-engine)
- ;; Create futures for parallel execution
- (let [futures (mapv #(future (search-fn % search-engine)) all-queries)
- result-lists (mapv deref futures)]
- (reciprocal-rank-fusion result-lists))))
+ (let [attempt (fn [query]
+ (try
+ {:ranked (search-fn query search-engine)}
+ (catch Exception e
+ {:error e})))]
+ ;; Zero queries case is handled nicely by the >1 branch
+ (if (= 1 (count all-queries))
+ [(attempt (first all-queries))]
+ ;; Create futures for parallel execution
+ (mapv deref (mapv #(future (attempt %)) all-queries)))))
(defn- confined-collection-id
"The collection that a search for `metabot` must stay inside, or nil. A confined metabot (embedded, or
@@ -339,14 +341,16 @@
`archived`, `collection-id` (numeric, scopes to the collection subtree; overrides the metabot's
own confined collection), `offset`. `filters-only?` makes a call with no queries run a single
nil-query search — a pure listing over the active filters — instead of returning nothing.
+ `partial-results?` keeps the other queries' results when a query's search fails, unless every
+ search failed; without it, any failure fails the call.
Each query fetches its full ranked pool (`ranked-results`), the pools are fused by rank, and the
fused ranking is paginated (`offset`/`limit`) exactly once (`search-results`) — so paging a
multi-query search is coherent. The result carries the size of the fused, deduped match set as
- `:total` metadata."
+ `:total` metadata, and the queries whose search failed as `:failed-queries`."
[{:keys [term-queries semantic-queries database-id created-at last-edited-at
entity-types limit metabot profile-id weights
- created-by archived collection-id offset filters-only?]}]
+ created-by archived collection-id offset filters-only? partial-results?]}]
(log/infof "[METABOT-SEARCH] Starting search with params: %s"
{:term-query-count (count term-queries)
:semantic-query-count (count semantic-queries)
@@ -414,30 +418,42 @@
(frequencies (map :model ranked)))
ranked))
ranked-fn* (fn [search-engine queries]
- (let [queries (search.engine/disjunction search-engine queries)]
- (join-results-by-rrf ranked-fn search-engine queries)))
+ (let [searches (search.engine/disjunction search-engine queries)
+ ;; A disjunction either keeps the queries apart or folds them all into one search.
+ covered (if (= 1 (count searches)) [queries] (map vector queries))]
+ (mapv #(assoc %1 :queries %2) (run-searches ranked-fn search-engine searches) covered)))
;; NOTE: if we add more semantic engines, e.g. 3rd party vector dbs, we'll need to make this more maintainable
semantic? #{:search.engine/semantic}
semantic-engine (u/seek semantic? (search.engine/active-engines))
fallback-engine (when semantic-engine
(search.engine/fallback-engine semantic-engine))
- fused-ranked (cond
+ engine-outcomes (cond
;; A pure listing over the filters: one search with no search string.
(and filters-only?
(empty? term-queries)
(empty? semantic-queries))
- (ranked-fn nil nil)
+ [[{:ranked (ranked-fn nil nil)}]]
;; Perform semantic and non-semantic search respectively, then fuse results.
semantic-engine
- (reciprocal-rank-fusion
- (map (fn [[engine queries]] (when (seq queries) (ranked-fn* engine queries)))
+ (into []
+ (keep (fn [[engine queries]] (when (seq queries) (ranked-fn* engine queries))))
{semantic-engine semantic-queries
- fallback-engine term-queries}))
+ fallback-engine term-queries})
;; Search for all the terms on equal footing, using the default engine.
:else
- (ranked-fn* nil (distinct (concat term-queries semantic-queries))))
+ [(ranked-fn* nil (distinct (concat term-queries semantic-queries)))])
+ outcomes (apply concat engine-outcomes)
+ failed (filter :error outcomes)
+ _ (when (seq failed)
+ (when (or (not partial-results?) (every? :error outcomes))
+ (throw (:error (first failed))))
+ (doseq [{:keys [error]} failed]
+ (log/error error "[METABOT-SEARCH] A search query failed; returning the other queries' results")))
+ fused-ranked (reciprocal-rank-fusion
+ (for [engine-outcome engine-outcomes]
+ (reciprocal-rank-fusion (keep :ranked engine-outcome))))
;; Paginate and hydrate the fused ranking exactly once. `search-results` slices to
;; [offset, offset+limit) and reports `:total` as the size of the full fused set — so the
;; total is knowable even under multi-query fusion, and only the returned page is hydrated.
@@ -463,7 +479,9 @@
enrich-with-metric-sources
enrich-with-measure-segment-base-tables
(validate-and-enrich-documents (boolean archived)))
- (vary-meta assoc :total total))))
+ (vary-meta assoc
+ :total total
+ :failed-queries (into [] (comp (mapcat :queries) (distinct)) failed)))))
(defn- table-refs->results
[ids]
@@ -570,22 +588,52 @@
enrich-with-measure-segment-base-tables
remove-unreadable-transforms)))
+(def ^:private default-search-limit 10)
+(def ^:private max-search-limit 50)
+
+(defn- search-restrictions
+ "What narrowed a search beyond its queries, as sentences for the model."
+ [default-types {:keys [metabot profile-id]} {:keys [entity_types]}]
+ (cond-> []
+ (and (seq entity_types) (not-every? (set entity_types) default-types))
+ (conj (str "Only these entity types were searched: " (str/join ", " (sort (set entity_types)))
+ ". If other types could match, add them to `entity_types`."))
+
+ (:use_verified_content metabot)
+ (conj "This Metabot searches only curated content, such as verified items and items in official collections.")
+
+ (confined-collection-id metabot profile-id)
+ (conj "This Metabot searches only inside its own collection.")))
+
(defn- format-search-output
- "Format search results as an LLM-ready string."
- [results]
- (let [results-xml (llm-shape/search-results->xml results)]
+ "Format search results as an LLM-ready string. `limit` is the page size the search asked for, and `restrictions`
+ are the [[search-restrictions]] that explain an empty result."
+ [results limit restrictions]
+ (let [results-xml (llm-shape/search-results->xml results)
+ {:keys [total failed-queries]} (meta results)]
(te/lines
"<result>"
results-xml
""
- (str "Total results: " (count results))
+ ;; Search caps each ranked pool, so the total is a lower bound.
+ (if (and total (< limit total))
+ (str "Showing " (count results) " of at least " total " results. "
+ "To see more, raise `limit` (max " max-search-limit ") or make the queries more specific.")
+ (str "Total results: " (count results)))
+ (when (seq failed-queries)
+ (str "The search failed for " (str/join ", " (map #(str "\"" (llm-shape/escape-xml %) "\"") failed-queries))
+ ", so these results may be incomplete. "
+ "If what you need isn't listed, tell the user the search was incomplete."))
+ (when (empty? results)
+ ["<no-results>"
+ restrictions
+ (str "Retry once with shorter `keyword_queries`: one or two words each, with synonyms as separate entries. "
+ "If that also finds nothing, tell the user instead of searching again.")
+ "</no-results>"])
"</result>"
"<instructions>"
instructions/search-result-instructions "</instructions>")))
-(def ^:private default-search-limit 10)
-(def ^:private max-search-limit 50)
-
;; Field-level descriptions surface to the model as JSON-Schema `description`s on the
;; tool's input parameters (via `malli.json-schema` in `metabase.metabot.self.claude`).
;; This is where per-parameter guidance lives; cross-tool search *strategy* (navigate
@@ -677,25 +725,31 @@
:recovery (conj (browse-steps scope) no-retry-step)})
(defn- do-search
- [default-types search-opts {:keys [semantic_queries keyword_queries entity_types limit]}]
+ [default-types search-opts {:keys [semantic_queries keyword_queries entity_types limit] :as args}]
;; Recoverable, not `:internal`: the agent can find the data without search, or tell the user that
;; search failed. `read_resource` does not use the search index.
- (let [opts (merge {:semantic-queries semantic_queries
- :term-queries keyword_queries
- :entity-types (or (seq entity_types) (vec default-types))
- :limit (min max-search-limit
- (or limit default-search-limit))}
- search-opts)
- results (try
- (search opts)
- (catch Exception e
- ;; The runtime logs a recoverable error at debug level, without its cause.
- (log/error e "Metabot search failed")
- (search-failed! (search-scope (:metabot opts) (:profile-id opts) (:database-id opts)))))]
- {:output (format-search-output results)
- :structured-output {:result-type :search
- :data results
- :total_count (count results)}
+ (let [limit (min max-search-limit (or limit default-search-limit))
+ opts (merge {:semantic-queries semantic_queries
+ :term-queries keyword_queries
+ :entity-types (or (seq entity_types) (vec default-types))
+ :limit limit
+ :partial-results? true}
+ search-opts)
+ results (try
+ (search opts)
+ (catch Exception e
+ ;; The runtime logs a recoverable error at debug level, without its cause.
+ (log/error e "Metabot search failed")
+ (search-failed! (search-scope (:metabot opts) (:profile-id opts) (:database-id opts)))))
+ restrictions (search-restrictions default-types search-opts args)]
+ {:output (format-search-output results limit restrictions)
+ :structured-output (cond-> {:result-type :search
+ :data results
+ :total_count (count results)}
+ (empty? results) (assoc :empty_reason (cond
+ (seq (:failed-queries (meta results))) "incomplete"
+ (seq restrictions) "restricted"
+ :else "no_match")))
:data-parts [(streaming/search-results-part
{:total_count (count results)
:results (mapv search-result->item results)})]}))
diff --git a/test/metabase/metabot/tools/search_test.clj b/test/metabase/metabot/tools/search_test.clj
index 4cb5ed3632e..d724e57735d 100644
--- a/test/metabase/metabot/tools/search_test.clj
+++ b/test/metabase/metabot/tools/search_test.clj
@@ -34,6 +34,40 @@
"Remove blank entries from every query list, then call `search` again "
"with non-blank queries in `keyword_queries`.")})
+(defn- do-with-search-stub!
+ "Call `thunk` with the engines set to `engines`, `ranked-results` answering each search string from
+ `string->ranked`, and `search-results` paginating whatever ranking it is handed. A search string that
+ `string->ranked` has no entry for throws, as a failing query would."
+ [engines string->ranked thunk]
+ (with-redefs [perms/impersonated-user? (constantly false)
+ perms/sandboxed-user? (constantly false)
+ search.engine/active-engines (constantly engines)
+ search.engine/disjunction (fn [_ terms] terms)
+ search-core/ranked-results (fn [{:keys [search-string]}]
+ (or (string->ranked search-string)
+ (throw (ex-info "Search index unavailable" {}))))
+ search-core/search-results (fn [{:keys [limit-int]} _ ranked]
+ {:data (vec (cond->> ranked limit-int (take limit-int)))
+ :total (count ranked)})]
+ (thunk)))
+
+(defn- result-notes
+ "The lines a search output adds after its results, up to `</result>`."
+ [output]
+ (->> (str/split-lines output)
+ (drop-while #(not= "</search-results>" %))
+ rest
+ (take-while #(not= "</result>" %))
+ (remove str/blank?)))
+
+(def ^:private failed-note
+ (str "The search failed for \"broken\", so these results may be incomplete. "
+ "If what you need isn't listed, tell the user the search was incomplete."))
+
+(def ^:private retry-note
+ (str "Retry once with shorter `keyword_queries`: one or two words each, with synonyms as separate entries. "
+ "If that also finds nothing, tell the user instead of searching again."))
+
(deftest ^:parallel search-requires-query-test
(mt/with-dynamic-fn-redefs [search/search (fn [_] (throw (ex-info "Search must not run" {})))]
(doseq [[tool-var extra-args] [[#'search/search-tool {}]
@@ -109,6 +143,68 @@
(invoke-search [tool-var] (merge extra-args queries) metabot "internal")))
(is (=? (merge {:metabot metabot} search-opts) @captured))))))))
+(deftest search-result-notes-test
+ (mt/with-test-user :rasta
+ (do-with-search-stub!
+ [:search.engine/appdb]
+ {"orders" (for [id [1 2 3]] {:id id :model "card" :name (str "Orders " id)})
+ "reports" (for [id (range 1 1001)] {:id id :model "card" :name (str "Report " id)})}
+ (fn []
+ (testing "a page cut short gives the total as a lower bound, since search caps the ranked pool"
+ (is (= ["Showing 1 of at least 1000 results. To see more, raise `limit` (max 50) or make the queries more specific."]
+ (result-notes (:output (invoke-search [#'search/search-tool]
+ {:keyword_queries ["reports"] :limit 1}))))))
+ (testing "a page that holds every match gives the total"
+ (is (= ["Total results: 3"]
+ (result-notes (:output (invoke-search [#'search/search-tool] {:keyword_queries ["orders"]}))))))
+ (testing "a query that fails is named, and the other query's results are kept"
+ (let [result (invoke-search [#'search/search-tool] {:keyword_queries ["orders" "broken"]})]
+ (is (nil? (:error result)))
+ (is (= [1 2 3] (map :id (get-in result [:structured-output :data]))))
+ (is (= ["Total results: 3" failed-note] (result-notes (:output result))))))
+ (testing "a search whose every query fails is a search failure the agent can recover from"
+ (is (= {:class :recoverable :code ::search/search-failed}
+ (:error (invoke-search [#'search/search-tool] {:keyword_queries ["broken" "also broken"]})))))))))
+
+(deftest empty-search-explains-itself-test
+ (let [explain (fn [tool-var args metabot]
+ (let [{:keys [output structured-output]} (invoke-search [tool-var] args metabot nil)]
+ {:empty_reason (:empty_reason structured-output)
+ :notes (result-notes output)}))]
+ (mt/with-dynamic-fn-redefs [search/search (constantly [])]
+ (testing "nothing narrowed the search"
+ (is (= {:empty_reason "no_match"
+ :notes ["Total results: 0" "<no-results>" retry-note "</no-results>"]}
+ (explain #'search/search-tool {:keyword_queries ["zzz"]} nil)))
+ (is (= {:empty_reason "no_match"
+ :notes ["Total results: 0" "<no-results>" retry-note "</no-results>"]}
+ (explain #'search/nlq-search-tool {:keyword_queries ["zzz"]
+ :entity_types (vec @#'search/general-entity-types)}
+ nil))
+ "every type the tool accepts narrows nothing, even beyond the tool's defaults"))
+ (testing "the entity types the model picked, curated content only and a confined collection are named"
+ (is (= {:empty_reason "restricted"
+ :notes ["Total results: 0"
+ "<no-results>"
+ "Only these entity types were searched: dashboard, metric. If other types could match, add them to `entity_types`."
+ "This Metabot searches only curated content, such as verified items and items in official collections."
+ "This Metabot searches only inside its own collection."
+ retry-note
+ "</no-results>"]}
+ (explain #'search/nlq-search-tool
+ {:keyword_queries ["zzz"] :entity_types ["metric" "dashboard"]}
+ {:collection_id 7 :use_verified_content true})))))
+ (mt/with-dynamic-fn-redefs [search/search (constantly (with-meta [] {:total 0 :failed-queries ["broken"]}))]
+ (testing "a failed query comes first among the reasons"
+ (is (= {:empty_reason "incomplete"
+ :notes ["Total results: 0"
+ failed-note
+ "<no-results>"
+ "This Metabot searches only curated content, such as verified items and items in official collections."
+ retry-note
+ "</no-results>"]}
+ (explain #'search/search-tool {:keyword_queries ["zzz" "broken"]} {:use_verified_content true})))))))
+
(deftest ^:parallel unsupported-search-entity-types-test
(mt/with-dynamic-fn-redefs [search/search (fn [_] (throw (ex-info "Search must not run" {})))]
(doseq [[tool-var extra-args entity-type]
@@ -292,6 +388,46 @@
(is (= 3 (:total (meta results))))
(is (= #{1 2 3} (set (map :id results)))))))))
+(deftest partial-results-test
+ (mt/with-test-user :rasta
+ (doseq [[engines queries] [[[:search.engine/appdb]
+ {:term-queries ["orders" "broken"]}]
+ [[:search.engine/semantic :search.engine/appdb]
+ {:term-queries ["orders"] :semantic-queries ["broken"]}]]]
+ (testing (str "engines " engines)
+ (do-with-search-stub!
+ engines
+ {"orders" [{:id 1 :model "card" :name "Orders"}]}
+ (fn []
+ (let [search* #(search/search (merge queries {:entity-types ["question"]} %))]
+ (testing "partial-results? keeps the other queries' results, lists the failed ones and logs the failure"
+ (mt/with-log-messages-for-level [logs [metabase.metabot.tools.search :error]]
+ (let [results (search* {:partial-results? true})]
+ (is (= [1] (map :id results)))
+ (is (= {:total 1 :failed-queries ["broken"]} (meta results)))
+ (is (=? [{:level :error :e #(= "Search index unavailable" (ex-message %))}] (logs))))))
+ (testing "without partial-results?, any failure fails the search"
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Search index unavailable" (search* {}))))
+ (testing "a search where every query fails still fails"
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Search index unavailable"
+ (search* {:term-queries ["broken" "also broken"]
+ :semantic-queries []
+ :partial-results? true})))))))))
+ (testing "a failed search that folded several queries into one lists every one of them"
+ (do-with-search-stub!
+ [:search.engine/semantic :search.engine/appdb]
+ {"orders" [{:id 1 :model "card" :name "Orders"}]}
+ (fn []
+ (with-redefs [search.engine/disjunction (fn [engine terms]
+ (if (= :search.engine/appdb engine)
+ [(str/join " OR " terms)]
+ terms))]
+ (is (= {:total 1 :failed-queries ["a" "b"]}
+ (meta (search/search {:semantic-queries ["orders"]
+ :term-queries ["a" "b"]
+ :entity-types ["question"]
+ :partial-results? true}))))))))))
+
(deftest ^:parallel postprocess-search-result-test
(testing "table result postprocessing"
(let [result {:model "table"