Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Tell Metabot why library search could not run (#85156)

Andrei Simionescu52280fd4198a
diff --git a/enterprise/backend/src/metabase_enterprise/search/entity_retrieval/core.clj b/enterprise/backend/src/metabase_enterprise/search/entity_retrieval/core.clj
index 44affd54c77..52278420c3d 100644
--- a/enterprise/backend/src/metabase_enterprise/search/entity_retrieval/core.clj
+++ b/enterprise/backend/src/metabase_enterprise/search/entity_retrieval/core.clj
@@ -140,7 +140,7 @@
;; Readiness can change after the tool is offered (for example settings can select an unavailable model).
;; Domain/configuration failures should make this optional retrieval path disappear, while JVM errors and
;; other unexpected exception types still escape.
- (log/warn e "configured embedding model could not be resolved; returning no library results")
+ (log/warn e "configured embedding model could not be resolved; library search is unavailable")
nil)))
(defn- missing-table-error?
@@ -403,23 +403,25 @@
blended score (cosine similarity plus a slight doc_type bump).
Each result is shaped `{:entity {:model :id} :doc_type :doc_text :score}`, best score first; the caller
dedupes the (many-per-entity) docs down to distinct entities.
- Returns [] when the pgvector store is unconfigured.
+ Returns [] for no matches, :busy during an index update, or :unavailable when the index or embedding
+ configuration cannot serve a search.
Hits are not permission-filtered — see [[metabase.search.entity-retrieval.mirror/search-unfiltered]] for what a
caller owes the current user."
:feature :library-retrieval
[user-search-prompt limit]
(if-not (available?)
- []
+ :unavailable
(let [pgvector (semantic.db.datasource/ensure-initialized-data-source!)
limit (or limit default-limit)
model (resolved-configured-model-for-search)]
;; Check compatibility under a short non-blocking read lock. Even metadata-table DDL can otherwise block this
;; preflight behind reconcile before we reach the protected query path. The later locked recheck remains
;; authoritative because a reconcile can begin while the embedding is generated.
- (if-not (and model
- (true? (reconcile/with-index-read-lock pgvector #(index-compatible? % model))))
- []
+ (case (when model (reconcile/with-index-read-lock pgvector #(index-compatible? % model)))
+ nil (if model :busy :unavailable)
+ false :unavailable
+ true
(let [embedding (embedding/get-embedding model (embedding/prefix-search-query model user-search-prompt)
{:type :query :record-tokens? true})
lit (index-table/format-embedding embedding)
@@ -431,7 +433,7 @@
;; The compatibility check and query share the reconcile lock and connection. A setting change
;; after the tool was offered must never query same-width vectors from another immutable space.
(if-not (index-compatible? conn model)
- []
+ :unavailable
(let [rows (try
(jdbc/execute!
conn
@@ -445,7 +447,7 @@
(sql/format {:quoted true}))
{:builder-fn jdbc.rs/as-unqualified-lower-maps})
(catch SQLException e
- ;; Only the index-not-ready states degrade to no results — the index, not the agent,
+ ;; Only the index-not-ready states report unavailable here: the index, not the agent,
;; is at fault and the next reconcile heals it:
;; 42P01 vectors table doesn't exist yet (pre-first-build; expected at boot, stay quiet)
;; 22000 stored vectors incompatible with the query vector (a dimension/format change
@@ -453,17 +455,19 @@
;; Any other SQL error — a connection loss or pgvector outage — propagates so the tool
;; reports the search as unavailable rather than as an empty library.
(case (.getSQLState e)
- "42P01" []
- "22000" (do (log/warnf "library entity index incompatible with the query vector; returning no results: %s" (ex-message e))
- [])
+ "42P01" :unavailable
+ "22000" (do (log/warnf "library entity index incompatible with the query vector: %s" (ex-message e))
+ :unavailable)
(do (analytics/inc! :metabase-entity-retrieval/search-failed)
(throw e)))))]
- (->> rows
- (map (fn [row]
- {:entity {:model (:entity_type row) :id (:entity_local_id row)}
- :doc_type (:doc_type row)
- :doc_text (:doc_text row)
- :score (score row)}))
- (sort-by (comp :total_score :score) >)
- vec)))))
- []))))))
+ (if (= :unavailable rows)
+ :unavailable
+ (->> rows
+ (map (fn [row]
+ {:entity {:model (:entity_type row) :id (:entity_local_id row)}
+ :doc_type (:doc_type row)
+ :doc_text (:doc_text row)
+ :score (score row)}))
+ (sort-by (comp :total_score :score) >)
+ vec))))))
+ :busy))))))
diff --git a/enterprise/backend/test/metabase_enterprise/search/entity_retrieval/core_test.clj b/enterprise/backend/test/metabase_enterprise/search/entity_retrieval/core_test.clj
index 95212c514b7..71ee80a128e 100644
--- a/enterprise/backend/test/metabase_enterprise/search/entity_retrieval/core_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/search/entity_retrieval/core_test.clj
@@ -90,9 +90,9 @@
reconcile/with-index-read-lock (fn [_ds f]
(f ::locked-connection))]
#_{:clj-kondo/ignore [:discouraged-var]}
- (is (= [] (entity-retrieval.core/search-unfiltered "the query" 10))))))
+ (is (= :unavailable (entity-retrieval.core/search-unfiltered "the query" 10))))))
-(deftest search-degrades-when-reconcile-holds-the-lock-test
+(deftest search-is-busy-when-reconcile-holds-the-lock-test
(mt/with-premium-features #{:library-retrieval}
(with-redefs [entity-retrieval.core/available? (constantly true)
semantic.db.datasource/ensure-initialized-data-source! (constantly ::datasource)
@@ -105,9 +105,9 @@
jdbc/execute! (fn [& _]
(throw (ex-info "must not query" {})))]
#_{:clj-kondo/ignore [:discouraged-var]}
- (is (= [] (entity-retrieval.core/search-unfiltered "the query" 10))))))
+ (is (= :busy (entity-retrieval.core/search-unfiltered "the query" 10))))))
-(deftest search-degrades-when-configured-model-cannot-be-resolved-test
+(deftest search-is-unavailable-when-configured-model-cannot-be-resolved-test
(mt/with-premium-features #{:library-retrieval}
(with-redefs [entity-retrieval.core/available? (constantly true)
semantic.db.datasource/ensure-initialized-data-source! (constantly ::datasource)
@@ -124,18 +124,52 @@
jdbc/execute! (fn [& _]
(throw (ex-info "must not query" {})))]
#_{:clj-kondo/ignore [:discouraged-var]}
- (is (= [] (entity-retrieval.core/search-unfiltered "the query" 10))))))
+ (is (= :unavailable (entity-retrieval.core/search-unfiltered "the query" 10))))))
(deftest dispatch-without-pgvector-test
(testing "with the feature enabled but pgvector unconfigured, the EE impls degrade gracefully"
- ;; Pin db-url to nil so the result is deterministic regardless of any ambient MB_PGVECTOR_DB_URL:
- ;; entity retrieval is dedicated-only, so no URL means available? is false, search returns [] and the
- ;; write-path nudge no-ops rather than throwing.
(mt/with-premium-features #{:library :library-retrieval}
(with-redefs [semantic.db.datasource/db-url nil]
- (is (= [] (search-unfiltered* "anything" 10)))
+ (is (= :unavailable (search-unfiltered* "anything" 10)))
(is (nil? (mirror/request-entity-sync! "table" 1)))))))
+(deftest search-index-states-test
+ (doseq [[scenario expected]
+ [[:query-lock :busy]
+ [:missing-index :unavailable]
+ [:changed-index :unavailable]
+ [:missing-vectors :unavailable]
+ [:incompatible-vectors :unavailable]
+ [:no-matches []]]]
+ (testing (name scenario)
+ (let [lock-count (atom 0)]
+ (mt/with-premium-features #{:library-retrieval}
+ (mt/with-dynamic-fn-redefs
+ [entity-retrieval.core/available? (constantly true)
+ semantic.db.datasource/ensure-initialized-data-source! (constantly ::datasource)
+ semantic.embedding/get-configured-model (constantly semantic.tu/mock-embedding-model)
+ reconcile/with-index-read-lock (fn [_ f]
+ (when-not (and (= :query-lock scenario)
+ (= 2 (swap! lock-count inc)))
+ (f ::connection)))
+ index-table/index-status (fn [& _]
+ (case scenario
+ :missing-index (throw (java.sql.SQLException.
+ "missing" "42P01"))
+ :changed-index (if (= 2 (swap! lock-count inc))
+ :incompatible
+ :compatible)
+ :compatible))
+ semantic.embedding/get-embedding (constantly [0.0 0.0 0.0 0.0])
+ jdbc/execute! (fn [& _]
+ (case scenario
+ :missing-vectors
+ (throw (java.sql.SQLException. "missing" "42P01"))
+ :incompatible-vectors
+ (throw (java.sql.SQLException. "dimensions" "22000"))
+ []))]
+ (is (= expected (search-unfiltered* "the query" 10)))))))))
+
(deftest targeted-drain-reconciles-each-dirty-entity-test
(testing "a targeted run snapshots and clears the dirty set, reconciling each entity once"
(let [dirty @#'entity-retrieval.core/dirty-entities
@@ -514,10 +548,8 @@
(index-table/vectors-table) ", "
(index-table/meta-table))]))))))))))))
-(deftest ^:synchronized search-degrades-on-dimension-mismatch-test
- ;; A post-upgrade embedding-dimension change leaves the query vector incompatible with the index column
- ;; until the next reconcile rebuilds it; search must degrade to [] rather than throw in that window.
- (testing "an index/query vector dimension mismatch degrades search to [] instead of throwing"
+(deftest ^:synchronized search-is-unavailable-on-dimension-mismatch-test
+ (testing "an index/query vector dimension mismatch reports unavailable"
(when semantic.db.datasource/db-url
(mt/with-premium-features #{:library :library-retrieval}
(let [suffix (System/nanoTime)
@@ -534,24 +566,19 @@
(try
(semantic.tu/with-mock-embeddings {"orders" [1.0 0.0 0.0 0.0]}
(reconcile/reconcile! ds (constantly semantic.tu/mock-embedding-model)))
- ;; Every entry point here is a defenterprise dispatcher that answers [] in OSS, so pair the
- ;; degrade assertion with a control: an unrun query and a degraded one both look like [].
#_{:clj-kondo/ignore [:discouraged-var]}
(semantic.tu/with-mock-embeddings {"orders" [1.0 0.0 0.0 0.0] "q" [1.0]}
(is (=? [{:model "table" :id table-id}]
(distinct (map :entity (entity-retrieval.core/search-unfiltered "orders" 10))))
- "the query really runs, so the [] below is the degrade path and not a fallback")
+ "the query really runs, so the unavailable result below comes from pgvector")
;; Shrink the configured model to one dimension, the way an upgrade would, keeping it
;; self-consistent so the provider resolves it instead of rejecting a changed space.
- ;; The meta row still describes the dim-4 build, so the compatibility preflight would
- ;; answer [] on its own; stub it aside to reach the case this test is about — a dim-1
- ;; literal meeting the vector(4) column, which is pgvector SQLState 22000.
(mt/with-dynamic-fn-redefs [semantic.embedding/get-configured-model
(constantly (semantic.tu/resolved-mock-embedding-model
:vector-dimensions 1))
index-table/index-status (constantly :compatible)]
(mt/with-log-messages-for-level [messages [metabase-enterprise.search.entity-retrieval.core :warn]]
- (is (= [] (entity-retrieval.core/search-unfiltered "q" 10)))
+ (is (= :unavailable (entity-retrieval.core/search-unfiltered "q" 10)))
(is (=? [{:level :warn
:message #"(?s)library entity index incompatible with the query vector.*"}]
(messages))
diff --git a/enterprise/backend/test/metabase_enterprise/search/entity_retrieval/logging_test.clj b/enterprise/backend/test/metabase_enterprise/search/entity_retrieval/logging_test.clj
new file mode 100644
index 00000000000..cb7c26878b0
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/search/entity_retrieval/logging_test.clj
@@ -0,0 +1,44 @@
+(ns metabase-enterprise.search.entity-retrieval.logging-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase-enterprise.search.entity-retrieval.core :as ee]
+ [metabase-enterprise.search.entity-retrieval.index-table :as index-table]
+ [metabase-enterprise.search.entity-retrieval.reconcile :as reconcile]
+ [metabase-enterprise.search.semantic.db.datasource :as datasource]
+ [metabase-enterprise.search.semantic.embedding :as embedding]
+ [metabase-enterprise.search.semantic.test-util :as semantic.tu]
+ [metabase.api-scope.core :as api-scope]
+ [metabase.metabot.scope :as scope]
+ [metabase.metabot.tools :as tools]
+ [metabase.test :as mt]
+ [next.jdbc :as jdbc]))
+
+(set! *warn-on-reflection* true)
+
+(deftest database-failure-retains-operator-diagnostics-test
+ (let [failure (java.sql.SQLException. "Connection lost" "08006")
+ entries (binding [scope/*current-user-scope* api-scope/unrestricted]
+ (tools/->entries [#'tools/retrieve-library-entities-tool] (atom {}) nil :nlq))]
+ (mt/with-premium-features #{:library-retrieval}
+ (mt/with-dynamic-fn-redefs
+ [ee/available? (constantly true)
+ datasource/ensure-initialized-data-source! (constantly ::ds)
+ embedding/get-configured-model (constantly semantic.tu/mock-embedding-model)
+ reconcile/with-index-read-lock (fn [_ f] (f ::connection))
+ index-table/index-status (constantly :compatible)
+ embedding/get-embedding (constantly [0.0 0.0 0.0 0.0])
+ jdbc/execute! (fn [& _] (throw failure))]
+ (mt/with-log-messages-for-level [messages ["metabase" :debug]]
+ (let [outcome ((:fn (get entries "retrieve_library_entities")) {:user_search_prompt "revenue"})
+ recorded (messages)]
+ (is (= {:output (str "Library search is unavailable.\n"
+ "Don't retry this call. If you can't find the data another way, tell the user "
+ "library search is unavailable.")
+ :error {:class :recoverable
+ :code :metabase.metabot.tools.entity-retrieval/library-search-unavailable}}
+ outcome))
+ (is (some (fn [{:keys [e level]}]
+ (and (#{:error :warn} level)
+ (some #(identical? failure %) (take-while some? (iterate ex-cause e)))))
+ recorded)
+ "The database exception must remain in operator logs while the model gets fixed text.")))))))
diff --git a/src/metabase/metabot/tools/entity_retrieval.clj b/src/metabase/metabot/tools/entity_retrieval.clj
index 312bea61706..e6f352e59fb 100644
--- a/src/metabase/metabot/tools/entity_retrieval.clj
+++ b/src/metabase/metabot/tools/entity_retrieval.clj
@@ -18,10 +18,12 @@
[clojure.string :as str]
[metabase.metabot.scope :as scope]
[metabase.metabot.tools.core :as tools]
+ [metabase.metabot.tools.error :as tools.error]
[metabase.metabot.tools.search :as tools.search]
[metabase.metabot.tools.shared.llm-shape :as llm-shape]
[metabase.search.entity-retrieval.core :as entity-retrieval]
- [metabase.util :as u]))
+ [metabase.util :as u]
+ [metabase.util.log :as log]))
(set! *warn-on-reflection* true)
@@ -84,16 +86,58 @@
[#{} []])
second))
-;; The one production read of the raw index — hence the lint exemption. Every field of every hit either is
-;; dropped here or survives the hydration below, which keeps only entities the current user can read.
+(def ^:private search-instead-step
+ {:uses #{"search"}
+ :text "Use `search` to find data for the user's request."})
+
+(tools/defrecoverable library-search-busy!
+ "The library index is being updated."
+ {:payload ::tools.search/search-scope}
+ [fallback-scope]
+ {:message "Library search is busy because the index is being updated."
+ :recovery (-> [{:uses #{"retrieve_library_entities"}
+ :text "Retry `retrieve_library_entities` once later."}
+ search-instead-step]
+ (into (tools.search/browse-steps fallback-scope))
+ (conj {:uses #{}
+ :text (str "If you can't find the data another way, tell the user library search is "
+ "being updated.")}))})
+
+(tools/defrecoverable library-search-unavailable!
+ "The library index or embedding provider cannot serve this search."
+ {:payload ::tools.search/search-scope}
+ [fallback-scope]
+ {:message "Library search is unavailable."
+ :recovery (-> [search-instead-step]
+ (into (tools.search/browse-steps fallback-scope))
+ (conj {:uses #{}
+ :text (str "Don't retry this call. If you can't find the data another way, tell the user "
+ "library search is unavailable.")}))})
+
+;; The one production read of the raw index, hence the lint exemption: hydration in [[build-matches]] drops the
+;; hits the current user can't read.
#_{:clj-kondo/ignore [:discouraged-var]}
+(defn- search-library [user-search-prompt n fallback-scope]
+ (let [result (try
+ (entity-retrieval/search-unfiltered user-search-prompt n)
+ (catch InterruptedException e
+ (throw e))
+ (catch Exception e
+ (log/error e "Library search failed")
+ (tools.error/throw-recoverable! ::library-search-unavailable fallback-scope {:cause e})))]
+ (case result
+ :busy (library-search-busy! fallback-scope)
+ :unavailable (library-search-unavailable! fallback-scope)
+ result)))
+
(defn- build-matches
"Fetch, dedupe to distinct entities, hydrate each match's entity ref into a full search record, and
- return the top `n` the current user can read.
+ return the top `n` the current user can read. `fallback-scope`, from [[tools.search/search-scope]], is what
+ the agent is told to browse instead when the fetch fails.
Each match: `{:doc_type :matched_text :usage_instructions :score :similarity :weak? :entity hydrated-hit}`."
- [user-search-prompt n]
+ [user-search-prompt n fallback-scope]
(let [fetch-n (min over-fetch-cap (* over-fetch-factor n))
- raw (entity-retrieval/search-unfiltered user-search-prompt fetch-n)
+ raw (search-library user-search-prompt fetch-n fallback-scope)
deduped (dedupe-by-entity raw)
refs (distinct (map :entity deduped))
;; Hydration permission-filters (entity-refs->search-results drops entities the user can't read),
@@ -144,7 +188,9 @@
(defn- format-output [matches]
(if (empty? matches)
- "<search_results>No matching library entities.</search_results>"
+ (str "<search_results>No matching library entities.</search_results>\n"
+ "<instructions>Rephrase `user_search_prompt` and call `retrieve_library_entities` once more. If there are "
+ "still no matches, ask the user to clarify or narrow the request.</instructions>")
(str (when (:weak? (first matches))
(str "<note>No strong library match for this request — treat the results below as weak "
"guesses; consider asking the user to clarify or narrow the request.</note>\n"))
@@ -174,10 +220,6 @@
{:message "`user_search_prompt` is blank. Set it to a full sentence that describes the data the user wants."
:recovery []})
-;; No capability gate: this tool lives only in the :nlq profile, which [[metabase.metabot.agent.profiles/
-;; get-profile]] serves (the library variant, with this tool) only when entity-retrieval-available? — otherwise it
-;; redirects to :nlq-fallback (general search). That single probe is the sole gate, so prompt and tools can't
-;; disagree about index availability.
(defrecord RetrieveLibraryEntitiesTool []
tools/Tool
(declaration [_]
@@ -196,11 +238,12 @@
:scope scope/agent-search
:args entity-retrieval-schema})
- (handle [_ {:keys [user_search_prompt limit]} _ctx]
+ (handle [_ {:keys [user_search_prompt limit]} {:keys [metabot profile-id]}]
(when (str/blank? user_search_prompt)
(blank-search-prompt! {}))
(let [n (min max-limit (or limit default-limit))
- matches (build-matches user_search_prompt n)]
+ matches (build-matches user_search_prompt n
+ (tools.search/search-scope metabot (some-> profile-id name) nil))]
{:output (format-output matches)
:structured-output {:result-type :search
:data (flatten-data matches)
diff --git a/src/metabase/metabot/tools/search.clj b/src/metabase/metabot/tools/search.clj
index 08636a883cf..1e225690251 100644
--- a/src/metabase/metabot/tools/search.clj
+++ b/src/metabase/metabot/tools/search.clj
@@ -21,7 +21,8 @@
[metabase.search.engine :as search.engine]
[metabase.transforms.core :as transforms]
[metabase.util :as u]
- [metabase.util.log :as log]))
+ [metabase.util.log :as log]
+ [metabase.util.malli.registry :as mr]))
(set! *warn-on-reflection* true)
@@ -623,6 +624,44 @@
{:uses #{"read_resource"}
:text (apply str "Find the data with `read_resource` instead: " uri-text)})
+(mr/def ::search-scope
+ "What a search reaches: the collection or database it is limited to, if any. `verified-only` is true when the
+ search is limited to verified content but `read_resource` is not, as in the profiles exempt from curation."
+ [:map {:closed true}
+ [:collection-id [:maybe :int]]
+ [:database-id [:maybe :int]]
+ [:verified-only :boolean]])
+
+(defn search-scope
+ "The [[::search-scope]] of a search for the Metabot row `metabot`.
+ `profile-id` names the profile the search runs under, and `database-id`, unless nil, is the database it is limited
+ to."
+ [metabot profile-id database-id]
+ {:collection-id (confined-collection-id metabot profile-id)
+ :database-id database-id
+ :verified-only (and (boolean (:use_verified_content metabot))
+ (not (shared/curated-only?)))})
+
+(defn browse-steps
+ "The recovery steps that find with `read_resource` what a search over a [[::search-scope]] could have found.
+ They browse only what the search was limited to, so a `verified-only` scope gets none."
+ [{:keys [collection-id database-id verified-only]}]
+ (cond
+ verified-only
+ []
+
+ collection-id
+ [(browse-instead-step "read `metabase://collection/" collection-id "/items`.")]
+
+ database-id
+ [(browse-instead-step "read `metabase://database/" database-id "/tables` and `metabase://database/"
+ database-id "/models`.")]
+
+ :else
+ [(browse-instead-step "read `metabase://collections` for metrics, models, questions and "
+ "dashboards, or `metabase://databases`, then `metabase://database/{id}/tables`, "
+ "for tables.")]))
+
(def ^:private no-retry-step
{:uses #{"search"}
:text (str "Do not call `search` again in this turn. If you cannot find the data another way, "
@@ -631,32 +670,11 @@
(tools/defrecoverable search-failed!
"The search backend failed, for example because the search index is missing.
- The browse step browses only the content that the failed search was limited to. `verified-only` is
- true when the search was limited to verified content but `read_resource` is not, as in the profiles
- exempt from curation; then there is no browse step. Every failure gets [[no-retry-step]], because
- each retry uses an iteration and logs a stack trace."
- {:payload [:map {:closed true}
- [:collection-id [:maybe :int]]
- [:database-id [:maybe :int]]
- [:verified-only :boolean]]}
- [{:keys [collection-id database-id verified-only]}]
+ Every failure gets [[no-retry-step]], because each retry uses an iteration and logs a stack trace."
+ {:payload ::search-scope}
+ [scope]
{:message "Search failed."
- :recovery (-> (cond
- verified-only
- []
-
- collection-id
- [(browse-instead-step "read `metabase://collection/" collection-id "/items`.")]
-
- database-id
- [(browse-instead-step "read `metabase://database/" database-id "/tables` and `metabase://database/"
- database-id "/models`.")]
-
- :else
- [(browse-instead-step "read `metabase://collections` for metrics, models, questions and "
- "dashboards, or `metabase://databases`, then `metabase://database/{id}/tables`, "
- "for tables.")])
- (conj no-retry-step))})
+ :recovery (conj (browse-steps scope) no-retry-step)})
(defn- do-search
[default-types search-opts {:keys [semantic_queries keyword_queries entity_types limit]}]
@@ -673,10 +691,7 @@
(catch Exception e
;; The runtime logs a recoverable error at debug level, without its cause.
(log/error e "Metabot search failed")
- (search-failed! {:collection-id (confined-collection-id (:metabot opts) (:profile-id opts))
- :database-id (:database-id opts)
- :verified-only (and (boolean (:use_verified_content (:metabot opts)))
- (not (shared/curated-only?)))})))]
+ (search-failed! (search-scope (:metabot opts) (:profile-id opts) (:database-id opts)))))]
{:output (format-search-output results)
:structured-output {:result-type :search
:data results
diff --git a/src/metabase/search/entity_retrieval/core.clj b/src/metabase/search/entity_retrieval/core.clj
index c4d66a6156a..939c826d199 100644
--- a/src/metabase/search/entity_retrieval/core.clj
+++ b/src/metabase/search/entity_retrieval/core.clj
@@ -4,7 +4,7 @@
The library entity index is a pgvector index of every library entity's name/description plus its OSI
`ai_context` synonyms/examples (see [[metabase.osi.models.osi-ai-context]]).
[[search-unfiltered]] matches a natural-language request against it by vector similarity (enterprise;
- returns [] in OSS). [[ai-context-instructions]] reads curator instructions live from the appdb.
+ returns :unavailable in OSS). [[ai-context-instructions]] reads curator instructions live from the appdb.
The index is user-agnostic, so [[search-unfiltered]] hits are not permission-filtered. A caller that
returns anything derived from them must first resolve each entity against the appdb and keep only the ones
diff --git a/src/metabase/search/entity_retrieval/mirror.clj b/src/metabase/search/entity_retrieval/mirror.clj
index f5f956598e1..5e22334f3be 100644
--- a/src/metabase/search/entity_retrieval/mirror.clj
+++ b/src/metabase/search/entity_retrieval/mirror.clj
@@ -4,7 +4,7 @@
The appdb (`osi_ai_context` + library membership) is authoritative; the index carries one embedding
per value-document and serves the `retrieve_library_entities` Metabot tool's similarity search.
These `defenterprise` shims route to [[metabase-enterprise.search.entity-retrieval.core]] when an enterprise
- license with library retrieval is present; otherwise they no-op or return empty."
+ license with library retrieval is present; otherwise searches are unavailable and writes no-op."
(:require
[metabase.premium-features.core :refer [defenterprise]]))
@@ -36,14 +36,15 @@
(defenterprise search-unfiltered
"Similarity-search the library entity index for the documents nearest `user-search-prompt`.
Returns up to `limit` matches shaped `{:entity {:model :id} :doc_type :doc_text :score}`,
- best score first; [] in OSS or when no pgvector store is configured.
+ best score first, or [] for no matches. Returns :busy when the index is being updated and :unavailable
+ when the index or embedding configuration cannot serve a search (including OSS).
The index is user-agnostic, so a hit's names, descriptions, synonyms, and examples may belong to an entity
the current user cannot read. Anything user-facing must first resolve each entity against the appdb, keep
only the readable ones, and drop hits that no longer resolve."
metabase-enterprise.search.entity-retrieval.core
[_user-search-prompt _limit]
- [])
+ :unavailable)
(defenterprise library-entity-keys
"Set of `[entity_type entity_local_id]` for every entity currently in the library; nil in OSS.
diff --git a/test/metabase/metabot/tools/entity_retrieval_test.clj b/test/metabase/metabot/tools/entity_retrieval_test.clj
index fabd928877c..74bea45d7ff 100644
--- a/test/metabase/metabot/tools/entity_retrieval_test.clj
+++ b/test/metabase/metabot/tools/entity_retrieval_test.clj
@@ -30,22 +30,35 @@
(defn- invoke
"Call `retrieve_library_entities` with `args` the way the agent loop does. Return the result, or
`{:output <the text the model reads> :error {:class :code}}` when the call fails."
- [args]
- (let [entries (binding [scope/*current-user-scope* api-scope/unrestricted]
- (metabot.tools/->entries [#'metabot.tools/retrieve-library-entities-tool] (atom {}) nil :nlq))]
- ((:fn (get entries "retrieve_library_entities")) args)))
+ ([args]
+ (invoke args [#'metabot.tools/retrieve-library-entities-tool]))
+ ([args tool-vars]
+ (invoke args tool-vars nil))
+ ([args tool-vars metabot]
+ (let [entries (binding [scope/*current-user-scope* api-scope/unrestricted]
+ (metabot.tools/->entries tool-vars (atom {}) metabot :nlq))]
+ ((:fn (get entries "retrieve_library_entities")) args))))
+
+(def ^:private nlq-discovery-tools
+ "The tools the `:nlq` profile finds data with."
+ [#'metabot.tools/retrieve-library-entities-tool #'metabot.tools/read-resource-tool])
+
+(def ^:private browse-instance-step
+ (str "Find the data with `read_resource` instead: read `metabase://collections` for metrics, models, questions "
+ "and dashboards, or `metabase://databases`, then `metabase://database/{id}/tables`, for tables."))
+
+(def ^:private unavailable-last-step
+ (str "Don't retry this call. If you can't find the data another way, tell the user library search is "
+ "unavailable."))
(deftest tool-shape-oss-fallback-test
- (testing "without the library-retrieval feature the tool returns the standard empty result shape"
+ (testing "without the library-retrieval feature the tool reports unavailable"
;; Pin the feature off so the defenterprise call takes its OSS fallback regardless of any ambient
;; premium token (e.g. an all-features dev token) or rows left in a local pgvector store.
(mt/with-premium-features #{}
- (is (=? {:output #(str/includes? % "No matching library entities")
- :structured-output {:result-type :search
- :data []
- :total_count 0
- :weak_match false}}
- (retrieve {:user_search_prompt "revenue per region"}))))))
+ (is (= {:output (str "Library search is unavailable.\n" unavailable-last-step)
+ :error {:class :recoverable :code ::entity-retrieval/library-search-unavailable}}
+ (invoke {:user_search_prompt "revenue per region"}))))))
(deftest similarity-helper-test
(let [similarity (var-get #'entity-retrieval/similarity)]
@@ -74,7 +87,7 @@
cs.core/ai-context-instructions (constantly {})
cs.core/library-entity-keys (constantly nil)]
(testing "asking for 2 returns the 2 readable entities (2 and 3), not just the 1 below the old top-2 cut"
- (is (= [2 3] (mapv (comp :id :entity) (build-matches "q" 2))))))))))
+ (is (= [2 3] (mapv (comp :id :entity) (build-matches "q" 2 (tools.search/search-scope nil nil nil)))))))))))
(deftest tool-never-exposes-unreadable-index-documents-test
(testing "neither output channel carries index text for an entity the current user can't read"
@@ -210,11 +223,53 @@
:usage_instructions "u2" :confidence "weak" :similarity 0.3}]
out)))))
-(deftest ^:parallel tool-search-failure-throws-test
- (testing "a failing library search propagates to the agent loop, which reports the call as failed"
+(deftest ^:parallel tool-search-unavailable-test
+ (testing "provider failures offer what the profile's search would reach, through the tools the profile has"
(with-unfiltered-search (fn [_ _] (throw (ex-info "Embedding service unavailable" {})))
- (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Embedding service unavailable"
- (retrieve {:user_search_prompt "revenue per region"}))))))
+ (doseq [[scenario tool-vars metabot steps]
+ [["without other discovery tools" [#'metabot.tools/retrieve-library-entities-tool] nil []]
+ [":nlq" nlq-discovery-tools nil [browse-instance-step]]
+ ["with `search`" (conj nlq-discovery-tools #'metabot.tools/search-tool) nil
+ ["Use `search` to find data for the user's request." browse-instance-step]]
+ [":nlq for a Metabot confined to a collection" nlq-discovery-tools {:collection_id 42}
+ ["Find the data with `read_resource` instead: read `metabase://collection/42/items`."]]
+ [":nlq for a Metabot limited to verified content" nlq-discovery-tools {:use_verified_content true}
+ []]]]
+ (testing scenario
+ (is (= {:output (str/join "\n" (concat ["Library search is unavailable."] steps [unavailable-last-step]))
+ :error {:class :recoverable :code ::entity-retrieval/library-search-unavailable}}
+ (invoke {:user_search_prompt "revenue per region"} tool-vars metabot))))))))
+
+(deftest ^:parallel interrupted-search-test
+ (with-unfiltered-search (fn [_ _] (throw (InterruptedException.)))
+ (is (thrown? InterruptedException (retrieve {:user_search_prompt "revenue per region"})))))
+
+(deftest ^:parallel tool-search-outcomes-test
+ (doseq [[result code output]
+ [[:busy ::entity-retrieval/library-search-busy
+ (str "Library search is busy because the index is being updated.\n"
+ "Retry `retrieve_library_entities` once later.\n"
+ browse-instance-step "\n"
+ "If you can't find the data another way, tell the user library search is being updated.")]
+ [:unavailable ::entity-retrieval/library-search-unavailable
+ (str "Library search is unavailable.\n" browse-instance-step "\n" unavailable-last-step)]]]
+ (testing (str result)
+ (with-unfiltered-search (constantly result)
+ (is (= {:output output :error {:class :recoverable :code code}}
+ (invoke {:user_search_prompt "revenue per region"} nlq-discovery-tools)))))))
+
+(deftest ^:parallel library-miss-test
+ (testing "a miss is a successful call that tells the model to rephrase once, then ask the user"
+ (with-unfiltered-search (constantly [])
+ (is (= {:output (str "<search_results>No matching library entities.</search_results>\n"
+ "<instructions>Rephrase `user_search_prompt` and call "
+ "`retrieve_library_entities` once more. If there are still no matches, ask the "
+ "user to clarify or narrow the request.</instructions>")
+ :structured-output {:result-type :search
+ :data []
+ :total_count 0
+ :weak_match false}}
+ (invoke {:user_search_prompt "revenue per region"}))))))
(deftest ^:parallel blank-search-prompt-test
(testing "a blank prompt goes back to the model before anything is searched"