The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Keep partial Metabot search results and explain empty ones (#85174)
Andrei Simionescub76292cc81b1
diff --git a/src/metabase/metabot/tools/search.clj b/src/metabase/metabot/tools/search.cljindex 1e225690251..8c1bc36940a 100644--- a/src/metabase/metabot/tools/search.clj+++ b/src/metabase/metabot/tools/search.clj@@ -310,18 +310,20 @@(sort-by :rrf >)(map :search-result)))))))-(defn- join-results-by-rrf- "Execute multiple search queries in parallel and combine results using Reciprocal Rank Fusion.- Items appearing in multiple result lists are boosted in the final ranking.- May return more results than requested limit."+(defn- run-searches+ "Execute multiple search queries in parallel. Returns one `{:ranked results}` per query, or `{:error e}` for a+ query whose search threw, so one failed query doesn't discard the others' results."[search-fn search-engine all-queries]- ;; Zero queries case is handled nicely by the >1 branch- (if (= 1 (count all-queries))- (search-fn (first all-queries) search-engine)- ;; Create futures for parallel execution- (let [futures (mapv #(future (search-fn % search-engine)) all-queries)- result-lists (mapv deref futures)]- (reciprocal-rank-fusion result-lists))))+ (let [attempt (fn [query]+ (try+ {:ranked (search-fn query search-engine)}+ (catch Exception e+ {:error e})))]+ ;; Zero queries case is handled nicely by the >1 branch+ (if (= 1 (count all-queries))+ [(attempt (first all-queries))]+ ;; Create futures for parallel execution+ (mapv deref (mapv #(future (attempt %)) all-queries)))))(defn- confined-collection-id"The collection that a search for `metabot` must stay inside, or nil. A confined metabot (embedded, or@@ -339,14 +341,16 @@`archived`, `collection-id` (numeric, scopes to the collection subtree; overrides the metabot'sown confined collection), `offset`. `filters-only?` makes a call with no queries run a singlenil-query search — a pure listing over the active filters — instead of returning nothing.+ `partial-results?` keeps the other queries' results when a query's search fails, unless every+ search failed; without it, any failure fails the call.Each query fetches its full ranked pool (`ranked-results`), the pools are fused by rank, and thefused ranking is paginated (`offset`/`limit`) exactly once (`search-results`) — so paging amulti-query search is coherent. The result carries the size of the fused, deduped match set as- `:total` metadata."+ `:total` metadata, and the queries whose search failed as `:failed-queries`."[{:keys [term-queries semantic-queries database-id created-at last-edited-atentity-types limit metabot profile-id weights- created-by archived collection-id offset filters-only?]}]+ created-by archived collection-id offset filters-only? partial-results?]}](log/infof "[METABOT-SEARCH] Starting search with params: %s"{:term-query-count (count term-queries):semantic-query-count (count semantic-queries)@@ -414,30 +418,42 @@(frequencies (map :model ranked)))ranked))ranked-fn* (fn [search-engine queries]- (let [queries (search.engine/disjunction search-engine queries)]- (join-results-by-rrf ranked-fn search-engine queries)))+ (let [searches (search.engine/disjunction search-engine queries)+ ;; A disjunction either keeps the queries apart or folds them all into one search.+ covered (if (= 1 (count searches)) [queries] (map vector queries))]+ (mapv #(assoc %1 :queries %2) (run-searches ranked-fn search-engine searches) covered)));; NOTE: if we add more semantic engines, e.g. 3rd party vector dbs, we'll need to make this more maintainablesemantic? #{:search.engine/semantic}semantic-engine (u/seek semantic? (search.engine/active-engines))fallback-engine (when semantic-engine(search.engine/fallback-engine semantic-engine))- fused-ranked (cond+ engine-outcomes (cond;; A pure listing over the filters: one search with no search string.(and filters-only?(empty? term-queries)(empty? semantic-queries))- (ranked-fn nil nil)+ [[{:ranked (ranked-fn nil nil)}]];; Perform semantic and non-semantic search respectively, then fuse results.semantic-engine- (reciprocal-rank-fusion- (map (fn [[engine queries]] (when (seq queries) (ranked-fn* engine queries)))+ (into []+ (keep (fn [[engine queries]] (when (seq queries) (ranked-fn* engine queries)))){semantic-engine semantic-queries- fallback-engine term-queries}))+ fallback-engine term-queries});; Search for all the terms on equal footing, using the default engine.:else- (ranked-fn* nil (distinct (concat term-queries semantic-queries))))+ [(ranked-fn* nil (distinct (concat term-queries semantic-queries)))])+ outcomes (apply concat engine-outcomes)+ failed (filter :error outcomes)+ _ (when (seq failed)+ (when (or (not partial-results?) (every? :error outcomes))+ (throw (:error (first failed))))+ (doseq [{:keys [error]} failed]+ (log/error error "[METABOT-SEARCH] A search query failed; returning the other queries' results")))+ fused-ranked (reciprocal-rank-fusion+ (for [engine-outcome engine-outcomes]+ (reciprocal-rank-fusion (keep :ranked engine-outcome))));; Paginate and hydrate the fused ranking exactly once. `search-results` slices to;; [offset, offset+limit) and reports `:total` as the size of the full fused set — so the;; total is knowable even under multi-query fusion, and only the returned page is hydrated.@@ -463,7 +479,9 @@enrich-with-metric-sourcesenrich-with-measure-segment-base-tables(validate-and-enrich-documents (boolean archived)))- (vary-meta assoc :total total))))+ (vary-meta assoc+ :total total+ :failed-queries (into [] (comp (mapcat :queries) (distinct)) failed)))))(defn- table-refs->results[ids]@@ -570,22 +588,52 @@enrich-with-measure-segment-base-tablesremove-unreadable-transforms)))+(def ^:private default-search-limit 10)+(def ^:private max-search-limit 50)++(defn- search-restrictions+ "What narrowed a search beyond its queries, as sentences for the model."+ [default-types {:keys [metabot profile-id]} {:keys [entity_types]}]+ (cond-> []+ (and (seq entity_types) (not-every? (set entity_types) default-types))+ (conj (str "Only these entity types were searched: " (str/join ", " (sort (set entity_types)))+ ". If other types could match, add them to `entity_types`."))++ (:use_verified_content metabot)+ (conj "This Metabot searches only curated content, such as verified items and items in official collections.")++ (confined-collection-id metabot profile-id)+ (conj "This Metabot searches only inside its own collection.")))+(defn- format-search-output- "Format search results as an LLM-ready string."- [results]- (let [results-xml (llm-shape/search-results->xml results)]+ "Format search results as an LLM-ready string. `limit` is the page size the search asked for, and `restrictions`+ are the [[search-restrictions]] that explain an empty result."+ [results limit restrictions]+ (let [results-xml (llm-shape/search-results->xml results)+ {:keys [total failed-queries]} (meta results)](te/lines"<result>"results-xml""- (str "Total results: " (count results))+ ;; Search caps each ranked pool, so the total is a lower bound.+ (if (and total (< limit total))+ (str "Showing " (count results) " of at least " total " results. "+ "To see more, raise `limit` (max " max-search-limit ") or make the queries more specific.")+ (str "Total results: " (count results)))+ (when (seq failed-queries)+ (str "The search failed for " (str/join ", " (map #(str "\"" (llm-shape/escape-xml %) "\"") failed-queries))+ ", so these results may be incomplete. "+ "If what you need isn't listed, tell the user the search was incomplete."))+ (when (empty? results)+ ["<no-results>"+ restrictions+ (str "Retry once with shorter `keyword_queries`: one or two words each, with synonyms as separate entries. "+ "If that also finds nothing, tell the user instead of searching again.")+ "</no-results>"])"</result>""<instructions>"instructions/search-result-instructions "</instructions>")))-(def ^:private default-search-limit 10)-(def ^:private max-search-limit 50)-;; Field-level descriptions surface to the model as JSON-Schema `description`s on the;; tool's input parameters (via `malli.json-schema` in `metabase.metabot.self.claude`).;; This is where per-parameter guidance lives; cross-tool search *strategy* (navigate@@ -677,25 +725,31 @@:recovery (conj (browse-steps scope) no-retry-step)})(defn- do-search- [default-types search-opts {:keys [semantic_queries keyword_queries entity_types limit]}]+ [default-types search-opts {:keys [semantic_queries keyword_queries entity_types limit] :as args}];; Recoverable, not `:internal`: the agent can find the data without search, or tell the user that;; search failed. `read_resource` does not use the search index.- (let [opts (merge {:semantic-queries semantic_queries- :term-queries keyword_queries- :entity-types (or (seq entity_types) (vec default-types))- :limit (min max-search-limit- (or limit default-search-limit))}- search-opts)- results (try- (search opts)- (catch Exception e- ;; The runtime logs a recoverable error at debug level, without its cause.- (log/error e "Metabot search failed")- (search-failed! (search-scope (:metabot opts) (:profile-id opts) (:database-id opts)))))]- {:output (format-search-output results)- :structured-output {:result-type :search- :data results- :total_count (count results)}+ (let [limit (min max-search-limit (or limit default-search-limit))+ opts (merge {:semantic-queries semantic_queries+ :term-queries keyword_queries+ :entity-types (or (seq entity_types) (vec default-types))+ :limit limit+ :partial-results? true}+ search-opts)+ results (try+ (search opts)+ (catch Exception e+ ;; The runtime logs a recoverable error at debug level, without its cause.+ (log/error e "Metabot search failed")+ (search-failed! (search-scope (:metabot opts) (:profile-id opts) (:database-id opts)))))+ restrictions (search-restrictions default-types search-opts args)]+ {:output (format-search-output results limit restrictions)+ :structured-output (cond-> {:result-type :search+ :data results+ :total_count (count results)}+ (empty? results) (assoc :empty_reason (cond+ (seq (:failed-queries (meta results))) "incomplete"+ (seq restrictions) "restricted"+ :else "no_match"))):data-parts [(streaming/search-results-part{:total_count (count results):results (mapv search-result->item results)})]}))diff --git a/test/metabase/metabot/tools/search_test.clj b/test/metabase/metabot/tools/search_test.cljindex 4cb5ed3632e..d724e57735d 100644--- a/test/metabase/metabot/tools/search_test.clj+++ b/test/metabase/metabot/tools/search_test.clj@@ -34,6 +34,40 @@"Remove blank entries from every query list, then call `search` again ""with non-blank queries in `keyword_queries`.")})+(defn- do-with-search-stub!+ "Call `thunk` with the engines set to `engines`, `ranked-results` answering each search string from+ `string->ranked`, and `search-results` paginating whatever ranking it is handed. A search string that+ `string->ranked` has no entry for throws, as a failing query would."+ [engines string->ranked thunk]+ (with-redefs [perms/impersonated-user? (constantly false)+ perms/sandboxed-user? (constantly false)+ search.engine/active-engines (constantly engines)+ search.engine/disjunction (fn [_ terms] terms)+ search-core/ranked-results (fn [{:keys [search-string]}]+ (or (string->ranked search-string)+ (throw (ex-info "Search index unavailable" {}))))+ search-core/search-results (fn [{:keys [limit-int]} _ ranked]+ {:data (vec (cond->> ranked limit-int (take limit-int)))+ :total (count ranked)})]+ (thunk)))++(defn- result-notes+ "The lines a search output adds after its results, up to `</result>`."+ [output]+ (->> (str/split-lines output)+ (drop-while #(not= "</search-results>" %))+ rest+ (take-while #(not= "</result>" %))+ (remove str/blank?)))++(def ^:private failed-note+ (str "The search failed for \"broken\", so these results may be incomplete. "+ "If what you need isn't listed, tell the user the search was incomplete."))++(def ^:private retry-note+ (str "Retry once with shorter `keyword_queries`: one or two words each, with synonyms as separate entries. "+ "If that also finds nothing, tell the user instead of searching again."))+(deftest ^:parallel search-requires-query-test(mt/with-dynamic-fn-redefs [search/search (fn [_] (throw (ex-info "Search must not run" {})))](doseq [[tool-var extra-args] [[#'search/search-tool {}]@@ -109,6 +143,68 @@(invoke-search [tool-var] (merge extra-args queries) metabot "internal")))(is (=? (merge {:metabot metabot} search-opts) @captured))))))))+(deftest search-result-notes-test+ (mt/with-test-user :rasta+ (do-with-search-stub!+ [:search.engine/appdb]+ {"orders" (for [id [1 2 3]] {:id id :model "card" :name (str "Orders " id)})+ "reports" (for [id (range 1 1001)] {:id id :model "card" :name (str "Report " id)})}+ (fn []+ (testing "a page cut short gives the total as a lower bound, since search caps the ranked pool"+ (is (= ["Showing 1 of at least 1000 results. To see more, raise `limit` (max 50) or make the queries more specific."]+ (result-notes (:output (invoke-search [#'search/search-tool]+ {:keyword_queries ["reports"] :limit 1}))))))+ (testing "a page that holds every match gives the total"+ (is (= ["Total results: 3"]+ (result-notes (:output (invoke-search [#'search/search-tool] {:keyword_queries ["orders"]}))))))+ (testing "a query that fails is named, and the other query's results are kept"+ (let [result (invoke-search [#'search/search-tool] {:keyword_queries ["orders" "broken"]})]+ (is (nil? (:error result)))+ (is (= [1 2 3] (map :id (get-in result [:structured-output :data]))))+ (is (= ["Total results: 3" failed-note] (result-notes (:output result))))))+ (testing "a search whose every query fails is a search failure the agent can recover from"+ (is (= {:class :recoverable :code ::search/search-failed}+ (:error (invoke-search [#'search/search-tool] {:keyword_queries ["broken" "also broken"]})))))))))++(deftest empty-search-explains-itself-test+ (let [explain (fn [tool-var args metabot]+ (let [{:keys [output structured-output]} (invoke-search [tool-var] args metabot nil)]+ {:empty_reason (:empty_reason structured-output)+ :notes (result-notes output)}))]+ (mt/with-dynamic-fn-redefs [search/search (constantly [])]+ (testing "nothing narrowed the search"+ (is (= {:empty_reason "no_match"+ :notes ["Total results: 0" "<no-results>" retry-note "</no-results>"]}+ (explain #'search/search-tool {:keyword_queries ["zzz"]} nil)))+ (is (= {:empty_reason "no_match"+ :notes ["Total results: 0" "<no-results>" retry-note "</no-results>"]}+ (explain #'search/nlq-search-tool {:keyword_queries ["zzz"]+ :entity_types (vec @#'search/general-entity-types)}+ nil))+ "every type the tool accepts narrows nothing, even beyond the tool's defaults"))+ (testing "the entity types the model picked, curated content only and a confined collection are named"+ (is (= {:empty_reason "restricted"+ :notes ["Total results: 0"+ "<no-results>"+ "Only these entity types were searched: dashboard, metric. If other types could match, add them to `entity_types`."+ "This Metabot searches only curated content, such as verified items and items in official collections."+ "This Metabot searches only inside its own collection."+ retry-note+ "</no-results>"]}+ (explain #'search/nlq-search-tool+ {:keyword_queries ["zzz"] :entity_types ["metric" "dashboard"]}+ {:collection_id 7 :use_verified_content true})))))+ (mt/with-dynamic-fn-redefs [search/search (constantly (with-meta [] {:total 0 :failed-queries ["broken"]}))]+ (testing "a failed query comes first among the reasons"+ (is (= {:empty_reason "incomplete"+ :notes ["Total results: 0"+ failed-note+ "<no-results>"+ "This Metabot searches only curated content, such as verified items and items in official collections."+ retry-note+ "</no-results>"]}+ (explain #'search/search-tool {:keyword_queries ["zzz" "broken"]} {:use_verified_content true})))))))+(deftest ^:parallel unsupported-search-entity-types-test(mt/with-dynamic-fn-redefs [search/search (fn [_] (throw (ex-info "Search must not run" {})))](doseq [[tool-var extra-args entity-type]@@ -292,6 +388,46 @@(is (= 3 (:total (meta results))))(is (= #{1 2 3} (set (map :id results)))))))))+(deftest partial-results-test+ (mt/with-test-user :rasta+ (doseq [[engines queries] [[[:search.engine/appdb]+ {:term-queries ["orders" "broken"]}]+ [[:search.engine/semantic :search.engine/appdb]+ {:term-queries ["orders"] :semantic-queries ["broken"]}]]]+ (testing (str "engines " engines)+ (do-with-search-stub!+ engines+ {"orders" [{:id 1 :model "card" :name "Orders"}]}+ (fn []+ (let [search* #(search/search (merge queries {:entity-types ["question"]} %))]+ (testing "partial-results? keeps the other queries' results, lists the failed ones and logs the failure"+ (mt/with-log-messages-for-level [logs [metabase.metabot.tools.search :error]]+ (let [results (search* {:partial-results? true})]+ (is (= [1] (map :id results)))+ (is (= {:total 1 :failed-queries ["broken"]} (meta results)))+ (is (=? [{:level :error :e #(= "Search index unavailable" (ex-message %))}] (logs))))))+ (testing "without partial-results?, any failure fails the search"+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Search index unavailable" (search* {}))))+ (testing "a search where every query fails still fails"+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Search index unavailable"+ (search* {:term-queries ["broken" "also broken"]+ :semantic-queries []+ :partial-results? true})))))))))+ (testing "a failed search that folded several queries into one lists every one of them"+ (do-with-search-stub!+ [:search.engine/semantic :search.engine/appdb]+ {"orders" [{:id 1 :model "card" :name "Orders"}]}+ (fn []+ (with-redefs [search.engine/disjunction (fn [engine terms]+ (if (= :search.engine/appdb engine)+ [(str/join " OR " terms)]+ terms))]+ (is (= {:total 1 :failed-queries ["a" "b"]}+ (meta (search/search {:semantic-queries ["orders"]+ :term-queries ["a" "b"]+ :entity-types ["question"]+ :partial-results? true}))))))))))+(deftest ^:parallel postprocess-search-result-test(testing "table result postprocessing"(let [result {:model "table"