The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Tell Metabot why library search could not run (#85156)
Andrei Simionescu52280fd4198a
diff --git a/enterprise/backend/src/metabase_enterprise/search/entity_retrieval/core.clj b/enterprise/backend/src/metabase_enterprise/search/entity_retrieval/core.cljindex 44affd54c77..52278420c3d 100644--- a/enterprise/backend/src/metabase_enterprise/search/entity_retrieval/core.clj+++ b/enterprise/backend/src/metabase_enterprise/search/entity_retrieval/core.clj@@ -140,7 +140,7 @@;; Readiness can change after the tool is offered (for example settings can select an unavailable model).;; Domain/configuration failures should make this optional retrieval path disappear, while JVM errors and;; other unexpected exception types still escape.- (log/warn e "configured embedding model could not be resolved; returning no library results")+ (log/warn e "configured embedding model could not be resolved; library search is unavailable")nil)))(defn- missing-table-error?@@ -403,23 +403,25 @@blended score (cosine similarity plus a slight doc_type bump).Each result is shaped `{:entity {:model :id} :doc_type :doc_text :score}`, best score first; the callerdedupes the (many-per-entity) docs down to distinct entities.- Returns [] when the pgvector store is unconfigured.+ Returns [] for no matches, :busy during an index update, or :unavailable when the index or embedding+ configuration cannot serve a search.Hits are not permission-filtered — see [[metabase.search.entity-retrieval.mirror/search-unfiltered]] for what acaller owes the current user.":feature :library-retrieval[user-search-prompt limit](if-not (available?)- []+ :unavailable(let [pgvector (semantic.db.datasource/ensure-initialized-data-source!)limit (or limit default-limit)model (resolved-configured-model-for-search)];; Check compatibility under a short non-blocking read lock. Even metadata-table DDL can otherwise block this;; preflight behind reconcile before we reach the protected query path. The later locked recheck remains;; authoritative because a reconcile can begin while the embedding is generated.- (if-not (and model- (true? (reconcile/with-index-read-lock pgvector #(index-compatible? % model))))- []+ (case (when model (reconcile/with-index-read-lock pgvector #(index-compatible? % model)))+ nil (if model :busy :unavailable)+ false :unavailable+ true(let [embedding (embedding/get-embedding model (embedding/prefix-search-query model user-search-prompt){:type :query :record-tokens? true})lit (index-table/format-embedding embedding)@@ -431,7 +433,7 @@;; The compatibility check and query share the reconcile lock and connection. A setting change;; after the tool was offered must never query same-width vectors from another immutable space.(if-not (index-compatible? conn model)- []+ :unavailable(let [rows (try(jdbc/execute!conn@@ -445,7 +447,7 @@(sql/format {:quoted true})){:builder-fn jdbc.rs/as-unqualified-lower-maps})(catch SQLException e- ;; Only the index-not-ready states degrade to no results — the index, not the agent,+ ;; Only the index-not-ready states report unavailable here: the index, not the agent,;; is at fault and the next reconcile heals it:;; 42P01 vectors table doesn't exist yet (pre-first-build; expected at boot, stay quiet);; 22000 stored vectors incompatible with the query vector (a dimension/format change@@ -453,17 +455,19 @@;; Any other SQL error — a connection loss or pgvector outage — propagates so the tool;; reports the search as unavailable rather than as an empty library.(case (.getSQLState e)- "42P01" []- "22000" (do (log/warnf "library entity index incompatible with the query vector; returning no results: %s" (ex-message e))- [])+ "42P01" :unavailable+ "22000" (do (log/warnf "library entity index incompatible with the query vector: %s" (ex-message e))+ :unavailable)(do (analytics/inc! :metabase-entity-retrieval/search-failed)(throw e)))))]- (->> rows- (map (fn [row]- {:entity {:model (:entity_type row) :id (:entity_local_id row)}- :doc_type (:doc_type row)- :doc_text (:doc_text row)- :score (score row)}))- (sort-by (comp :total_score :score) >)- vec)))))- []))))))+ (if (= :unavailable rows)+ :unavailable+ (->> rows+ (map (fn [row]+ {:entity {:model (:entity_type row) :id (:entity_local_id row)}+ :doc_type (:doc_type row)+ :doc_text (:doc_text row)+ :score (score row)}))+ (sort-by (comp :total_score :score) >)+ vec))))))+ :busy))))))diff --git a/enterprise/backend/test/metabase_enterprise/search/entity_retrieval/core_test.clj b/enterprise/backend/test/metabase_enterprise/search/entity_retrieval/core_test.cljindex 95212c514b7..71ee80a128e 100644--- a/enterprise/backend/test/metabase_enterprise/search/entity_retrieval/core_test.clj+++ b/enterprise/backend/test/metabase_enterprise/search/entity_retrieval/core_test.clj@@ -90,9 +90,9 @@reconcile/with-index-read-lock (fn [_ds f](f ::locked-connection))]#_{:clj-kondo/ignore [:discouraged-var]}- (is (= [] (entity-retrieval.core/search-unfiltered "the query" 10))))))+ (is (= :unavailable (entity-retrieval.core/search-unfiltered "the query" 10))))))-(deftest search-degrades-when-reconcile-holds-the-lock-test+(deftest search-is-busy-when-reconcile-holds-the-lock-test(mt/with-premium-features #{:library-retrieval}(with-redefs [entity-retrieval.core/available? (constantly true)semantic.db.datasource/ensure-initialized-data-source! (constantly ::datasource)@@ -105,9 +105,9 @@jdbc/execute! (fn [& _](throw (ex-info "must not query" {})))]#_{:clj-kondo/ignore [:discouraged-var]}- (is (= [] (entity-retrieval.core/search-unfiltered "the query" 10))))))+ (is (= :busy (entity-retrieval.core/search-unfiltered "the query" 10))))))-(deftest search-degrades-when-configured-model-cannot-be-resolved-test+(deftest search-is-unavailable-when-configured-model-cannot-be-resolved-test(mt/with-premium-features #{:library-retrieval}(with-redefs [entity-retrieval.core/available? (constantly true)semantic.db.datasource/ensure-initialized-data-source! (constantly ::datasource)@@ -124,18 +124,52 @@jdbc/execute! (fn [& _](throw (ex-info "must not query" {})))]#_{:clj-kondo/ignore [:discouraged-var]}- (is (= [] (entity-retrieval.core/search-unfiltered "the query" 10))))))+ (is (= :unavailable (entity-retrieval.core/search-unfiltered "the query" 10))))))(deftest dispatch-without-pgvector-test(testing "with the feature enabled but pgvector unconfigured, the EE impls degrade gracefully"- ;; Pin db-url to nil so the result is deterministic regardless of any ambient MB_PGVECTOR_DB_URL:- ;; entity retrieval is dedicated-only, so no URL means available? is false, search returns [] and the- ;; write-path nudge no-ops rather than throwing.(mt/with-premium-features #{:library :library-retrieval}(with-redefs [semantic.db.datasource/db-url nil]- (is (= [] (search-unfiltered* "anything" 10)))+ (is (= :unavailable (search-unfiltered* "anything" 10)))(is (nil? (mirror/request-entity-sync! "table" 1)))))))+(deftest search-index-states-test+ (doseq [[scenario expected]+ [[:query-lock :busy]+ [:missing-index :unavailable]+ [:changed-index :unavailable]+ [:missing-vectors :unavailable]+ [:incompatible-vectors :unavailable]+ [:no-matches []]]]+ (testing (name scenario)+ (let [lock-count (atom 0)]+ (mt/with-premium-features #{:library-retrieval}+ (mt/with-dynamic-fn-redefs+ [entity-retrieval.core/available? (constantly true)+ semantic.db.datasource/ensure-initialized-data-source! (constantly ::datasource)+ semantic.embedding/get-configured-model (constantly semantic.tu/mock-embedding-model)+ reconcile/with-index-read-lock (fn [_ f]+ (when-not (and (= :query-lock scenario)+ (= 2 (swap! lock-count inc)))+ (f ::connection)))+ index-table/index-status (fn [& _]+ (case scenario+ :missing-index (throw (java.sql.SQLException.+ "missing" "42P01"))+ :changed-index (if (= 2 (swap! lock-count inc))+ :incompatible+ :compatible)+ :compatible))+ semantic.embedding/get-embedding (constantly [0.0 0.0 0.0 0.0])+ jdbc/execute! (fn [& _]+ (case scenario+ :missing-vectors+ (throw (java.sql.SQLException. "missing" "42P01"))+ :incompatible-vectors+ (throw (java.sql.SQLException. "dimensions" "22000"))+ []))]+ (is (= expected (search-unfiltered* "the query" 10)))))))))+(deftest targeted-drain-reconciles-each-dirty-entity-test(testing "a targeted run snapshots and clears the dirty set, reconciling each entity once"(let [dirty @#'entity-retrieval.core/dirty-entities@@ -514,10 +548,8 @@(index-table/vectors-table) ", "(index-table/meta-table))]))))))))))))-(deftest ^:synchronized search-degrades-on-dimension-mismatch-test- ;; A post-upgrade embedding-dimension change leaves the query vector incompatible with the index column- ;; until the next reconcile rebuilds it; search must degrade to [] rather than throw in that window.- (testing "an index/query vector dimension mismatch degrades search to [] instead of throwing"+(deftest ^:synchronized search-is-unavailable-on-dimension-mismatch-test+ (testing "an index/query vector dimension mismatch reports unavailable"(when semantic.db.datasource/db-url(mt/with-premium-features #{:library :library-retrieval}(let [suffix (System/nanoTime)@@ -534,24 +566,19 @@(try(semantic.tu/with-mock-embeddings {"orders" [1.0 0.0 0.0 0.0]}(reconcile/reconcile! ds (constantly semantic.tu/mock-embedding-model)))- ;; Every entry point here is a defenterprise dispatcher that answers [] in OSS, so pair the- ;; degrade assertion with a control: an unrun query and a degraded one both look like [].#_{:clj-kondo/ignore [:discouraged-var]}(semantic.tu/with-mock-embeddings {"orders" [1.0 0.0 0.0 0.0] "q" [1.0]}(is (=? [{:model "table" :id table-id}](distinct (map :entity (entity-retrieval.core/search-unfiltered "orders" 10))))- "the query really runs, so the [] below is the degrade path and not a fallback")+ "the query really runs, so the unavailable result below comes from pgvector");; Shrink the configured model to one dimension, the way an upgrade would, keeping it;; self-consistent so the provider resolves it instead of rejecting a changed space.- ;; The meta row still describes the dim-4 build, so the compatibility preflight would- ;; answer [] on its own; stub it aside to reach the case this test is about — a dim-1- ;; literal meeting the vector(4) column, which is pgvector SQLState 22000.(mt/with-dynamic-fn-redefs [semantic.embedding/get-configured-model(constantly (semantic.tu/resolved-mock-embedding-model:vector-dimensions 1))index-table/index-status (constantly :compatible)](mt/with-log-messages-for-level [messages [metabase-enterprise.search.entity-retrieval.core :warn]]- (is (= [] (entity-retrieval.core/search-unfiltered "q" 10)))+ (is (= :unavailable (entity-retrieval.core/search-unfiltered "q" 10)))(is (=? [{:level :warn:message #"(?s)library entity index incompatible with the query vector.*"}](messages))diff --git a/enterprise/backend/test/metabase_enterprise/search/entity_retrieval/logging_test.clj b/enterprise/backend/test/metabase_enterprise/search/entity_retrieval/logging_test.cljnew file mode 100644index 00000000000..cb7c26878b0--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/search/entity_retrieval/logging_test.clj@@ -0,0 +1,44 @@+(ns metabase-enterprise.search.entity-retrieval.logging-test+ (:require+ [clojure.test :refer :all]+ [metabase-enterprise.search.entity-retrieval.core :as ee]+ [metabase-enterprise.search.entity-retrieval.index-table :as index-table]+ [metabase-enterprise.search.entity-retrieval.reconcile :as reconcile]+ [metabase-enterprise.search.semantic.db.datasource :as datasource]+ [metabase-enterprise.search.semantic.embedding :as embedding]+ [metabase-enterprise.search.semantic.test-util :as semantic.tu]+ [metabase.api-scope.core :as api-scope]+ [metabase.metabot.scope :as scope]+ [metabase.metabot.tools :as tools]+ [metabase.test :as mt]+ [next.jdbc :as jdbc]))++(set! *warn-on-reflection* true)++(deftest database-failure-retains-operator-diagnostics-test+ (let [failure (java.sql.SQLException. "Connection lost" "08006")+ entries (binding [scope/*current-user-scope* api-scope/unrestricted]+ (tools/->entries [#'tools/retrieve-library-entities-tool] (atom {}) nil :nlq))]+ (mt/with-premium-features #{:library-retrieval}+ (mt/with-dynamic-fn-redefs+ [ee/available? (constantly true)+ datasource/ensure-initialized-data-source! (constantly ::ds)+ embedding/get-configured-model (constantly semantic.tu/mock-embedding-model)+ reconcile/with-index-read-lock (fn [_ f] (f ::connection))+ index-table/index-status (constantly :compatible)+ embedding/get-embedding (constantly [0.0 0.0 0.0 0.0])+ jdbc/execute! (fn [& _] (throw failure))]+ (mt/with-log-messages-for-level [messages ["metabase" :debug]]+ (let [outcome ((:fn (get entries "retrieve_library_entities")) {:user_search_prompt "revenue"})+ recorded (messages)]+ (is (= {:output (str "Library search is unavailable.\n"+ "Don't retry this call. If you can't find the data another way, tell the user "+ "library search is unavailable.")+ :error {:class :recoverable+ :code :metabase.metabot.tools.entity-retrieval/library-search-unavailable}}+ outcome))+ (is (some (fn [{:keys [e level]}]+ (and (#{:error :warn} level)+ (some #(identical? failure %) (take-while some? (iterate ex-cause e)))))+ recorded)+ "The database exception must remain in operator logs while the model gets fixed text.")))))))diff --git a/src/metabase/metabot/tools/entity_retrieval.clj b/src/metabase/metabot/tools/entity_retrieval.cljindex 312bea61706..e6f352e59fb 100644--- a/src/metabase/metabot/tools/entity_retrieval.clj+++ b/src/metabase/metabot/tools/entity_retrieval.clj@@ -18,10 +18,12 @@[clojure.string :as str][metabase.metabot.scope :as scope][metabase.metabot.tools.core :as tools]+ [metabase.metabot.tools.error :as tools.error][metabase.metabot.tools.search :as tools.search][metabase.metabot.tools.shared.llm-shape :as llm-shape][metabase.search.entity-retrieval.core :as entity-retrieval]- [metabase.util :as u]))+ [metabase.util :as u]+ [metabase.util.log :as log]))(set! *warn-on-reflection* true)@@ -84,16 +86,58 @@[#{} []])second))-;; The one production read of the raw index — hence the lint exemption. Every field of every hit either is-;; dropped here or survives the hydration below, which keeps only entities the current user can read.+(def ^:private search-instead-step+ {:uses #{"search"}+ :text "Use `search` to find data for the user's request."})++(tools/defrecoverable library-search-busy!+ "The library index is being updated."+ {:payload ::tools.search/search-scope}+ [fallback-scope]+ {:message "Library search is busy because the index is being updated."+ :recovery (-> [{:uses #{"retrieve_library_entities"}+ :text "Retry `retrieve_library_entities` once later."}+ search-instead-step]+ (into (tools.search/browse-steps fallback-scope))+ (conj {:uses #{}+ :text (str "If you can't find the data another way, tell the user library search is "+ "being updated.")}))})++(tools/defrecoverable library-search-unavailable!+ "The library index or embedding provider cannot serve this search."+ {:payload ::tools.search/search-scope}+ [fallback-scope]+ {:message "Library search is unavailable."+ :recovery (-> [search-instead-step]+ (into (tools.search/browse-steps fallback-scope))+ (conj {:uses #{}+ :text (str "Don't retry this call. If you can't find the data another way, tell the user "+ "library search is unavailable.")}))})++;; The one production read of the raw index, hence the lint exemption: hydration in [[build-matches]] drops the+;; hits the current user can't read.#_{:clj-kondo/ignore [:discouraged-var]}+(defn- search-library [user-search-prompt n fallback-scope]+ (let [result (try+ (entity-retrieval/search-unfiltered user-search-prompt n)+ (catch InterruptedException e+ (throw e))+ (catch Exception e+ (log/error e "Library search failed")+ (tools.error/throw-recoverable! ::library-search-unavailable fallback-scope {:cause e})))]+ (case result+ :busy (library-search-busy! fallback-scope)+ :unavailable (library-search-unavailable! fallback-scope)+ result)))+(defn- build-matches"Fetch, dedupe to distinct entities, hydrate each match's entity ref into a full search record, and- return the top `n` the current user can read.+ return the top `n` the current user can read. `fallback-scope`, from [[tools.search/search-scope]], is what+ the agent is told to browse instead when the fetch fails.Each match: `{:doc_type :matched_text :usage_instructions :score :similarity :weak? :entity hydrated-hit}`."- [user-search-prompt n]+ [user-search-prompt n fallback-scope](let [fetch-n (min over-fetch-cap (* over-fetch-factor n))- raw (entity-retrieval/search-unfiltered user-search-prompt fetch-n)+ raw (search-library user-search-prompt fetch-n fallback-scope)deduped (dedupe-by-entity raw)refs (distinct (map :entity deduped));; Hydration permission-filters (entity-refs->search-results drops entities the user can't read),@@ -144,7 +188,9 @@(defn- format-output [matches](if (empty? matches)- "<search_results>No matching library entities.</search_results>"+ (str "<search_results>No matching library entities.</search_results>\n"+ "<instructions>Rephrase `user_search_prompt` and call `retrieve_library_entities` once more. If there are "+ "still no matches, ask the user to clarify or narrow the request.</instructions>")(str (when (:weak? (first matches))(str "<note>No strong library match for this request — treat the results below as weak ""guesses; consider asking the user to clarify or narrow the request.</note>\n"))@@ -174,10 +220,6 @@{:message "`user_search_prompt` is blank. Set it to a full sentence that describes the data the user wants.":recovery []})-;; No capability gate: this tool lives only in the :nlq profile, which [[metabase.metabot.agent.profiles/-;; get-profile]] serves (the library variant, with this tool) only when entity-retrieval-available? — otherwise it-;; redirects to :nlq-fallback (general search). That single probe is the sole gate, so prompt and tools can't-;; disagree about index availability.(defrecord RetrieveLibraryEntitiesTool []tools/Tool(declaration [_]@@ -196,11 +238,12 @@:scope scope/agent-search:args entity-retrieval-schema})- (handle [_ {:keys [user_search_prompt limit]} _ctx]+ (handle [_ {:keys [user_search_prompt limit]} {:keys [metabot profile-id]}](when (str/blank? user_search_prompt)(blank-search-prompt! {}))(let [n (min max-limit (or limit default-limit))- matches (build-matches user_search_prompt n)]+ matches (build-matches user_search_prompt n+ (tools.search/search-scope metabot (some-> profile-id name) nil))]{:output (format-output matches):structured-output {:result-type :search:data (flatten-data matches)diff --git a/src/metabase/metabot/tools/search.clj b/src/metabase/metabot/tools/search.cljindex 08636a883cf..1e225690251 100644--- a/src/metabase/metabot/tools/search.clj+++ b/src/metabase/metabot/tools/search.clj@@ -21,7 +21,8 @@[metabase.search.engine :as search.engine][metabase.transforms.core :as transforms][metabase.util :as u]- [metabase.util.log :as log]))+ [metabase.util.log :as log]+ [metabase.util.malli.registry :as mr]))(set! *warn-on-reflection* true)@@ -623,6 +624,44 @@{:uses #{"read_resource"}:text (apply str "Find the data with `read_resource` instead: " uri-text)})+(mr/def ::search-scope+ "What a search reaches: the collection or database it is limited to, if any. `verified-only` is true when the+ search is limited to verified content but `read_resource` is not, as in the profiles exempt from curation."+ [:map {:closed true}+ [:collection-id [:maybe :int]]+ [:database-id [:maybe :int]]+ [:verified-only :boolean]])++(defn search-scope+ "The [[::search-scope]] of a search for the Metabot row `metabot`.+ `profile-id` names the profile the search runs under, and `database-id`, unless nil, is the database it is limited+ to."+ [metabot profile-id database-id]+ {:collection-id (confined-collection-id metabot profile-id)+ :database-id database-id+ :verified-only (and (boolean (:use_verified_content metabot))+ (not (shared/curated-only?)))})++(defn browse-steps+ "The recovery steps that find with `read_resource` what a search over a [[::search-scope]] could have found.+ They browse only what the search was limited to, so a `verified-only` scope gets none."+ [{:keys [collection-id database-id verified-only]}]+ (cond+ verified-only+ []++ collection-id+ [(browse-instead-step "read `metabase://collection/" collection-id "/items`.")]++ database-id+ [(browse-instead-step "read `metabase://database/" database-id "/tables` and `metabase://database/"+ database-id "/models`.")]++ :else+ [(browse-instead-step "read `metabase://collections` for metrics, models, questions and "+ "dashboards, or `metabase://databases`, then `metabase://database/{id}/tables`, "+ "for tables.")]))+(def ^:private no-retry-step{:uses #{"search"}:text (str "Do not call `search` again in this turn. If you cannot find the data another way, "@@ -631,32 +670,11 @@(tools/defrecoverable search-failed!"The search backend failed, for example because the search index is missing.- The browse step browses only the content that the failed search was limited to. `verified-only` is- true when the search was limited to verified content but `read_resource` is not, as in the profiles- exempt from curation; then there is no browse step. Every failure gets [[no-retry-step]], because- each retry uses an iteration and logs a stack trace."- {:payload [:map {:closed true}- [:collection-id [:maybe :int]]- [:database-id [:maybe :int]]- [:verified-only :boolean]]}- [{:keys [collection-id database-id verified-only]}]+ Every failure gets [[no-retry-step]], because each retry uses an iteration and logs a stack trace."+ {:payload ::search-scope}+ [scope]{:message "Search failed."- :recovery (-> (cond- verified-only- []-- collection-id- [(browse-instead-step "read `metabase://collection/" collection-id "/items`.")]-- database-id- [(browse-instead-step "read `metabase://database/" database-id "/tables` and `metabase://database/"- database-id "/models`.")]-- :else- [(browse-instead-step "read `metabase://collections` for metrics, models, questions and "- "dashboards, or `metabase://databases`, then `metabase://database/{id}/tables`, "- "for tables.")])- (conj no-retry-step))})+ :recovery (conj (browse-steps scope) no-retry-step)})(defn- do-search[default-types search-opts {:keys [semantic_queries keyword_queries entity_types limit]}]@@ -673,10 +691,7 @@(catch Exception e;; The runtime logs a recoverable error at debug level, without its cause.(log/error e "Metabot search failed")- (search-failed! {:collection-id (confined-collection-id (:metabot opts) (:profile-id opts))- :database-id (:database-id opts)- :verified-only (and (boolean (:use_verified_content (:metabot opts)))- (not (shared/curated-only?)))})))]+ (search-failed! (search-scope (:metabot opts) (:profile-id opts) (:database-id opts)))))]{:output (format-search-output results):structured-output {:result-type :search:data resultsdiff --git a/src/metabase/search/entity_retrieval/core.clj b/src/metabase/search/entity_retrieval/core.cljindex c4d66a6156a..939c826d199 100644--- a/src/metabase/search/entity_retrieval/core.clj+++ b/src/metabase/search/entity_retrieval/core.clj@@ -4,7 +4,7 @@The library entity index is a pgvector index of every library entity's name/description plus its OSI`ai_context` synonyms/examples (see [[metabase.osi.models.osi-ai-context]]).[[search-unfiltered]] matches a natural-language request against it by vector similarity (enterprise;- returns [] in OSS). [[ai-context-instructions]] reads curator instructions live from the appdb.+ returns :unavailable in OSS). [[ai-context-instructions]] reads curator instructions live from the appdb.The index is user-agnostic, so [[search-unfiltered]] hits are not permission-filtered. A caller thatreturns anything derived from them must first resolve each entity against the appdb and keep only the onesdiff --git a/src/metabase/search/entity_retrieval/mirror.clj b/src/metabase/search/entity_retrieval/mirror.cljindex f5f956598e1..5e22334f3be 100644--- a/src/metabase/search/entity_retrieval/mirror.clj+++ b/src/metabase/search/entity_retrieval/mirror.clj@@ -4,7 +4,7 @@The appdb (`osi_ai_context` + library membership) is authoritative; the index carries one embeddingper value-document and serves the `retrieve_library_entities` Metabot tool's similarity search.These `defenterprise` shims route to [[metabase-enterprise.search.entity-retrieval.core]] when an enterprise- license with library retrieval is present; otherwise they no-op or return empty."+ license with library retrieval is present; otherwise searches are unavailable and writes no-op."(:require[metabase.premium-features.core :refer [defenterprise]]))@@ -36,14 +36,15 @@(defenterprise search-unfiltered"Similarity-search the library entity index for the documents nearest `user-search-prompt`.Returns up to `limit` matches shaped `{:entity {:model :id} :doc_type :doc_text :score}`,- best score first; [] in OSS or when no pgvector store is configured.+ best score first, or [] for no matches. Returns :busy when the index is being updated and :unavailable+ when the index or embedding configuration cannot serve a search (including OSS).The index is user-agnostic, so a hit's names, descriptions, synonyms, and examples may belong to an entitythe current user cannot read. Anything user-facing must first resolve each entity against the appdb, keeponly the readable ones, and drop hits that no longer resolve."metabase-enterprise.search.entity-retrieval.core[_user-search-prompt _limit]- [])+ :unavailable)(defenterprise library-entity-keys"Set of `[entity_type entity_local_id]` for every entity currently in the library; nil in OSS.diff --git a/test/metabase/metabot/tools/entity_retrieval_test.clj b/test/metabase/metabot/tools/entity_retrieval_test.cljindex fabd928877c..74bea45d7ff 100644--- a/test/metabase/metabot/tools/entity_retrieval_test.clj+++ b/test/metabase/metabot/tools/entity_retrieval_test.clj@@ -30,22 +30,35 @@(defn- invoke"Call `retrieve_library_entities` with `args` the way the agent loop does. Return the result, or`{:output <the text the model reads> :error {:class :code}}` when the call fails."- [args]- (let [entries (binding [scope/*current-user-scope* api-scope/unrestricted]- (metabot.tools/->entries [#'metabot.tools/retrieve-library-entities-tool] (atom {}) nil :nlq))]- ((:fn (get entries "retrieve_library_entities")) args)))+ ([args]+ (invoke args [#'metabot.tools/retrieve-library-entities-tool]))+ ([args tool-vars]+ (invoke args tool-vars nil))+ ([args tool-vars metabot]+ (let [entries (binding [scope/*current-user-scope* api-scope/unrestricted]+ (metabot.tools/->entries tool-vars (atom {}) metabot :nlq))]+ ((:fn (get entries "retrieve_library_entities")) args))))++(def ^:private nlq-discovery-tools+ "The tools the `:nlq` profile finds data with."+ [#'metabot.tools/retrieve-library-entities-tool #'metabot.tools/read-resource-tool])++(def ^:private browse-instance-step+ (str "Find the data with `read_resource` instead: read `metabase://collections` for metrics, models, questions "+ "and dashboards, or `metabase://databases`, then `metabase://database/{id}/tables`, for tables."))++(def ^:private unavailable-last-step+ (str "Don't retry this call. If you can't find the data another way, tell the user library search is "+ "unavailable."))(deftest tool-shape-oss-fallback-test- (testing "without the library-retrieval feature the tool returns the standard empty result shape"+ (testing "without the library-retrieval feature the tool reports unavailable";; Pin the feature off so the defenterprise call takes its OSS fallback regardless of any ambient;; premium token (e.g. an all-features dev token) or rows left in a local pgvector store.(mt/with-premium-features #{}- (is (=? {:output #(str/includes? % "No matching library entities")- :structured-output {:result-type :search- :data []- :total_count 0- :weak_match false}}- (retrieve {:user_search_prompt "revenue per region"}))))))+ (is (= {:output (str "Library search is unavailable.\n" unavailable-last-step)+ :error {:class :recoverable :code ::entity-retrieval/library-search-unavailable}}+ (invoke {:user_search_prompt "revenue per region"}))))))(deftest similarity-helper-test(let [similarity (var-get #'entity-retrieval/similarity)]@@ -74,7 +87,7 @@cs.core/ai-context-instructions (constantly {})cs.core/library-entity-keys (constantly nil)](testing "asking for 2 returns the 2 readable entities (2 and 3), not just the 1 below the old top-2 cut"- (is (= [2 3] (mapv (comp :id :entity) (build-matches "q" 2))))))))))+ (is (= [2 3] (mapv (comp :id :entity) (build-matches "q" 2 (tools.search/search-scope nil nil nil)))))))))))(deftest tool-never-exposes-unreadable-index-documents-test(testing "neither output channel carries index text for an entity the current user can't read"@@ -210,11 +223,53 @@:usage_instructions "u2" :confidence "weak" :similarity 0.3}]out)))))-(deftest ^:parallel tool-search-failure-throws-test- (testing "a failing library search propagates to the agent loop, which reports the call as failed"+(deftest ^:parallel tool-search-unavailable-test+ (testing "provider failures offer what the profile's search would reach, through the tools the profile has"(with-unfiltered-search (fn [_ _] (throw (ex-info "Embedding service unavailable" {})))- (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Embedding service unavailable"- (retrieve {:user_search_prompt "revenue per region"}))))))+ (doseq [[scenario tool-vars metabot steps]+ [["without other discovery tools" [#'metabot.tools/retrieve-library-entities-tool] nil []]+ [":nlq" nlq-discovery-tools nil [browse-instance-step]]+ ["with `search`" (conj nlq-discovery-tools #'metabot.tools/search-tool) nil+ ["Use `search` to find data for the user's request." browse-instance-step]]+ [":nlq for a Metabot confined to a collection" nlq-discovery-tools {:collection_id 42}+ ["Find the data with `read_resource` instead: read `metabase://collection/42/items`."]]+ [":nlq for a Metabot limited to verified content" nlq-discovery-tools {:use_verified_content true}+ []]]]+ (testing scenario+ (is (= {:output (str/join "\n" (concat ["Library search is unavailable."] steps [unavailable-last-step]))+ :error {:class :recoverable :code ::entity-retrieval/library-search-unavailable}}+ (invoke {:user_search_prompt "revenue per region"} tool-vars metabot))))))))++(deftest ^:parallel interrupted-search-test+ (with-unfiltered-search (fn [_ _] (throw (InterruptedException.)))+ (is (thrown? InterruptedException (retrieve {:user_search_prompt "revenue per region"})))))++(deftest ^:parallel tool-search-outcomes-test+ (doseq [[result code output]+ [[:busy ::entity-retrieval/library-search-busy+ (str "Library search is busy because the index is being updated.\n"+ "Retry `retrieve_library_entities` once later.\n"+ browse-instance-step "\n"+ "If you can't find the data another way, tell the user library search is being updated.")]+ [:unavailable ::entity-retrieval/library-search-unavailable+ (str "Library search is unavailable.\n" browse-instance-step "\n" unavailable-last-step)]]]+ (testing (str result)+ (with-unfiltered-search (constantly result)+ (is (= {:output output :error {:class :recoverable :code code}}+ (invoke {:user_search_prompt "revenue per region"} nlq-discovery-tools)))))))++(deftest ^:parallel library-miss-test+ (testing "a miss is a successful call that tells the model to rephrase once, then ask the user"+ (with-unfiltered-search (constantly [])+ (is (= {:output (str "<search_results>No matching library entities.</search_results>\n"+ "<instructions>Rephrase `user_search_prompt` and call "+ "`retrieve_library_entities` once more. If there are still no matches, ask the "+ "user to clarify or narrow the request.</instructions>")+ :structured-output {:result-type :search+ :data []+ :total_count 0+ :weak_match false}}+ (invoke {:user_search_prompt "revenue per region"}))))))(deftest ^:parallel blank-search-prompt-test(testing "a blank prompt goes back to the model before anything is searched"