Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Decouple library-retrieval and semantic-search features (#76872)

Chris Truter7ed335db9c54
diff --git a/enterprise/backend/src/metabase_enterprise/entity_retrieval/core.clj b/enterprise/backend/src/metabase_enterprise/entity_retrieval/core.clj
index 4c9898603dc..086872b2b5c 100644
--- a/enterprise/backend/src/metabase_enterprise/entity_retrieval/core.clj
+++ b/enterprise/backend/src/metabase_enterprise/entity_retrieval/core.clj
@@ -32,17 +32,26 @@
(defn pgvector-configured?
"True when a pgvector store is configured (read once at boot from MB_PGVECTOR_DB_URL).
The sync task gates scheduling on this rather than [[available?]], so the periodic safety net exists
- even when the semantic-search feature is turned on after startup (a common onboarding flow)."
+ even when the library-retrieval feature is turned on after startup (a common onboarding flow)."
[]
(string? (not-empty semantic.db.datasource/db-url)))
(defn available?
- "Whether the entity-retrieval mirror can run right now: a pgvector store is configured and the license
- includes semantic search.
- The feature check can flip at runtime (token entered post-boot), so callers re-evaluate per use."
+ "Whether the entity-retrieval mirror can run right now. All four must hold:
+ - a pgvector store is configured (somewhere to hold the index),
+ - the license includes `:library-retrieval` (the tool is entitled),
+ - the license includes `:library`: retrieval operates over library entities, so the tool is meaningless
+ without it (the index would have nothing to hold) — enforced here so a token granting
+ `:library-retrieval` alone degrades to \"unavailable\" explicitly, not silently via an empty index,
+ - an embedding backend is configured (see [[embedding/embedding-supported?]]): without one, both
+ reconcile and query embedding would throw, so we gate rather than fail mid-run.
+ The feature and config checks can flip at runtime (token/settings entered post-boot), so callers
+ re-evaluate per use."
[]
(and (pgvector-configured?)
- (premium-features/has-feature? :semantic-search)))
+ (premium-features/has-feature? :library)
+ (premium-features/has-feature? :library-retrieval)
+ (embedding/embedding-supported? (embedding/get-configured-model))))
(defn- index-ready?
"Whether the library entity index can serve a query right now: its meta row matches the configured
@@ -60,14 +69,14 @@
(seq (jdbc/execute! ds [(format "SELECT 1 FROM \"%s\" LIMIT 1" index-table/*vectors-table*)]))))
(catch Throwable _ false))))
-;; OSS-callable surface used to decide whether to OFFER the curated retrieve_library_entities tool: it must
+;; OSS-callable surface used to decide whether to OFFER the retrieve_library_entities tool: it must
;; be able to actually answer, so beyond config + license the index has to be built for the current model and
;; populated. (The write/reconcile path gates on the looser [[available?]] instead, so an empty or stale
;; index can still be rebuilt.) Runs unconditionally rather than gating on :feature — the OSS fallback
;; `false` already covers the unlicensed case.
(defenterprise entity-retrieval-available?
- "EE impl: pgvector configured + semantic-search licensed AND the index is ready (built for the current
- model and populated), so the curated tool is offered only when it can serve a query (otherwise the agent
+ "EE impl: pgvector configured + library-retrieval licensed AND the index is ready (built for the current
+ model and populated), so the tool is offered only when it can serve a query (otherwise the agent
gets the general-search fallback)."
:feature :none
[]
@@ -181,7 +190,7 @@
this call finishes. Returns {:index {:inserted n :deleted n :unchanged n} :execution {:waited_ms _
:ran_ms _}}, or nil when entity retrieval isn't available (no pgvector store configured).
See [[reconcile-full-coalesced!]] for the coalescing semantics it shares with the periodic backstop."
- :feature :semantic-search
+ :feature :library-retrieval
[]
(when (available?)
(reconcile-full-coalesced!)))
@@ -191,7 +200,7 @@
Fire-and-forget: never blocks, throws, or does embedding/pgvector work on the calling (appdb-write)
thread — the reconcile runs later on a future. Covers `osi_ai_context` edits; membership / name /
description changes to the underlying entity are caught by the periodic full reconcile."
- :feature :semantic-search
+ :feature :library-retrieval
[entity-type entity-local-id]
(when (available?)
(try
@@ -203,7 +212,7 @@
(defenterprise library-entity-keys
"Live set of `[entity_type entity_local_id]` for entities currently in the library (see the OSS shim)."
- :feature :semantic-search
+ :feature :library-retrieval
[]
(reconcile/library-entity-keys))
@@ -245,7 +254,7 @@
Each result is shaped `{:entity {:model :id} :doc_type :doc_text :score}`, best score first; the caller
dedupes the (many-per-entity) docs down to distinct entities.
Returns [] when the pgvector store is unconfigured."
- :feature :semantic-search
+ :feature :library-retrieval
[user-search-prompt limit]
(if-not (available?)
[]
@@ -263,7 +272,7 @@
[[:raw distance] :distance])
(sql.helpers/from (keyword index-table/*vectors-table*))
;; Exact scan, no HNSW: the blended order-by can't use an ANN index, and the
- ;; curated set is tiny.
+ ;; library set is tiny.
(sql.helpers/order-by [[:raw (ranking-sql distance)] :asc])
(sql.helpers/limit limit)
(sql/format {:quoted true}))
diff --git a/enterprise/backend/src/metabase_enterprise/entity_retrieval/reconcile.clj b/enterprise/backend/src/metabase_enterprise/entity_retrieval/reconcile.clj
index 0a4cc8ef2af..009c0aba038 100644
--- a/enterprise/backend/src/metabase_enterprise/entity_retrieval/reconcile.clj
+++ b/enterprise/backend/src/metabase_enterprise/entity_retrieval/reconcile.clj
@@ -324,7 +324,7 @@
;; Scalability — targeted writes, full-diff backstop.
;; The full diff is O(total index size) per run, not O(changes). Embeddings — the only expensive resource —
;; are already change-scoped (only a new doc_text is embedded; an idle run embeds nothing), and the O(total)
-;; metadata read is cheap over the *library* (the bounded curated tier), so a full run stays comfortable
+;; metadata read is cheap over the *library* (a bounded tier), so a full run stays comfortable
;; into the tens of thousands of docs. The write path no longer pays it on every edit: an `osi_ai_context`
;; write drives [[reconcile-entity!]] (one entity's slice), and [[reconcile!]] runs only on a slow schedule
;; and from the force-reconcile API as the backstop for membership / name / description changes that aren't
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/embedding.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/embedding.clj
index d797230569d..c3e06f03c80 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/embedding.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/embedding.clj
@@ -402,6 +402,28 @@
:model-name (semantic-settings/ee-embedding-model)
:vector-dimensions (semantic-settings/ee-embedding-model-dimensions)})
+(defmulti embedding-supported?
+ "Whether `embedding-model`'s provider is *configured* to compute embeddings — the endpoint/credentials it
+ needs are present. This is a config-presence check, not a liveness probe: a set URL whose service is down
+ (or a stopped ollama) still reads as supported and surfaces at call time. Dispatches on provider,
+ mirroring [[get-embedding]] and the config each provider's impl resolves; a new provider — including a
+ future in-process embedder — adds a method. The `:default` is false, so an unrecognized provider gates
+ callers off safely."
+ {:arglists '([embedding-model])} dispatch-provider)
+
+(defmethod embedding-supported? :default [_] false)
+
+(defmethod embedding-supported? "ai-service" [_]
+ (boolean (or (not-empty (semantic-settings/ee-embedding-service-base-url))
+ (not-empty (llm.settings/ai-service-base-url)))))
+
+(defmethod embedding-supported? "openai" [_]
+ (boolean (not-empty (semantic-settings/openai-api-key))))
+
+;; ollama's endpoint is hardcoded (localhost:11434) with no setting to check, so config-presence is always
+;; true — consistent with ai-service/openai, which likewise check for a configured URL, not a live server.
+(defmethod embedding-supported? "ollama" [_] true)
+
(defn- calc-token-metrics
[texts]
(let [counts (map count-tokens texts)
diff --git a/enterprise/backend/test/metabase_enterprise/api/session_test.clj b/enterprise/backend/test/metabase_enterprise/api/session_test.clj
index 0e138b35a54..2f3a45d5a93 100644
--- a/enterprise/backend/test/metabase_enterprise/api/session_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/api/session_test.clj
@@ -30,6 +30,7 @@
:disable-password-login
:database-auth-providers
:library
+ :library-retrieval
:development-mode
:email-allow-list
:email-restrict-recipients
@@ -82,6 +83,7 @@
:disable_password_login true
:database_auth_providers true
:library true
+ :library_retrieval true
:development_mode true
:email_allow_list true
:email_restrict_recipients true
diff --git a/enterprise/backend/test/metabase_enterprise/entity_retrieval/core_test.clj b/enterprise/backend/test/metabase_enterprise/entity_retrieval/core_test.clj
index 974231bcbea..7f7ea85b3ff 100644
--- a/enterprise/backend/test/metabase_enterprise/entity_retrieval/core_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/entity_retrieval/core_test.clj
@@ -35,7 +35,7 @@
(testing "with the feature enabled but pgvector unconfigured, the EE impls degrade gracefully"
;; Pin db-url to nil so the result is deterministic regardless of any ambient MB_PGVECTOR_DB_URL:
;; available? is false, so search returns [] and the write-path nudge no-ops rather than throwing.
- (mt/with-premium-features #{:semantic-search}
+ (mt/with-premium-features #{:library :library-retrieval}
(with-redefs [semantic.db.datasource/db-url nil]
(is (= [] (mirror/search "anything" 10)))
(is (nil? (mirror/request-entity-sync! "table" 1)))))))
@@ -48,7 +48,7 @@
(reset! dirty #{["table" 7] ["metric" 9]})
(mt/with-dynamic-fn-redefs
[semantic.db.datasource/ensure-initialized-data-source! (constantly ::ds)
- semantic.embedding/get-configured-model (constantly ::model)
+ semantic.embedding/get-configured-model (constantly semantic.tu/mock-embedding-model)
reconcile/reconcile-entity! (fn [_ds _model entity-type entity-local-id]
(swap! reconciled conj [entity-type entity-local-id])
{:inserted 1 :deleted 0 :unchanged 0})]
@@ -58,7 +58,7 @@
(deftest request-entity-sync-is-fire-and-forget-test
(testing "request-entity-sync! returns nil without throwing on the calling thread, even if the reconcile fails"
- (mt/with-premium-features #{:semantic-search}
+ (mt/with-premium-features #{:library :library-retrieval}
(with-redefs [semantic.db.datasource/db-url "jdbc:postgresql://stub"]
(let [tc @#'entity-retrieval.core/targeted-current
tn @#'entity-retrieval.core/targeted-next
@@ -66,7 +66,7 @@
(reset! tc nil) (reset! tn nil) (reset! dirty #{})
(mt/with-dynamic-fn-redefs
[semantic.db.datasource/ensure-initialized-data-source! (constantly ::ds)
- semantic.embedding/get-configured-model (constantly ::model)
+ semantic.embedding/get-configured-model (constantly semantic.tu/mock-embedding-model)
reconcile/reconcile-entity! (fn [& _] (throw (ex-info "boom" {})))]
(is (nil? (entity-retrieval.core/request-entity-sync! "table" 1)))
;; let the fire-and-forget drain settle so its logged error doesn't bleed into later tests
@@ -75,7 +75,7 @@
(deftest force-reconcile-coalescing-test
(testing "force-reconcile! never joins the in-flight run, queues one shared follow-up, and reports index + timing"
- (mt/with-premium-features #{:semantic-search}
+ (mt/with-premium-features #{:library :library-retrieval}
;; db-url is read directly as a var, so with-redefs (not with-dynamic-fn-redefs) is required here.
(with-redefs [semantic.db.datasource/db-url "jdbc:postgresql://stub"]
(let [current @#'entity-retrieval.core/full-current
@@ -83,7 +83,7 @@
calls (atom 0)]
(mt/with-dynamic-fn-redefs
[semantic.db.datasource/ensure-initialized-data-source! (constantly ::ds)
- semantic.embedding/get-configured-model (constantly ::model)
+ semantic.embedding/get-configured-model (constantly semantic.tu/mock-embedding-model)
reconcile/reconcile! (fn [_ds _resolve-model]
(swap! calls inc)
{:inserted 3 :deleted 1 :unchanged 2})]
@@ -112,24 +112,43 @@
(deftest force-reconcile-unavailable-returns-nil-test
(testing "force-reconcile! is nil (so the API can 400) when pgvector isn't configured"
- (mt/with-premium-features #{:semantic-search}
+ (mt/with-premium-features #{:library :library-retrieval}
(with-redefs [semantic.db.datasource/db-url nil]
(is (nil? (entity-retrieval.core/force-reconcile!)))))))
-(deftest pgvector-configured-decoupled-from-feature-test
- (testing "scheduling gates on pgvector config, not the feature flag, so a post-boot license still syncs"
- ;; db-url is read directly as a var, so with-redefs (not with-dynamic-fn-redefs) is required here.
- (with-redefs [semantic.db.datasource/db-url "jdbc:postgresql://stub"]
- (mt/with-premium-features #{}
- (is (true? (entity-retrieval.core/pgvector-configured?)))
- (is (false? (entity-retrieval.core/available?))))
- (mt/with-premium-features #{:semantic-search}
- (is (true? (entity-retrieval.core/available?)))))))
+(deftest available?-gates-test
+ (testing "available? requires pgvector + :library + :library-retrieval + a configured embedder; each gate alone is decisive"
+ ;; The mock model's provider is "mock" -> embedding-supported? true, so these cases isolate the other gates.
+ ;; db-url is read directly as a var, so with-redefs (not with-dynamic-fn-redefs) is required for it.
+ (mt/with-dynamic-fn-redefs [semantic.embedding/get-configured-model (constantly semantic.tu/mock-embedding-model)]
+ (with-redefs [semantic.db.datasource/db-url "jdbc:postgresql://stub"]
+ (mt/with-premium-features #{}
+ (is (true? (entity-retrieval.core/pgvector-configured?))
+ "scheduling gates on pgvector config, not the license, so a post-boot license still syncs")
+ (is (false? (entity-retrieval.core/available?))))
+ (mt/with-premium-features #{:library-retrieval}
+ (is (false? (entity-retrieval.core/available?))
+ ":library-retrieval alone is not enough — retrieval also requires the :library feature"))
+ (mt/with-premium-features #{:library}
+ (is (false? (entity-retrieval.core/available?))
+ ":library without :library-retrieval does not entitle the tool"))
+ (mt/with-premium-features #{:library :library-retrieval}
+ (is (true? (entity-retrieval.core/available?)))))
+ ;; pgvector unconfigured -> unavailable regardless of license
+ (with-redefs [semantic.db.datasource/db-url nil]
+ (mt/with-premium-features #{:library :library-retrieval}
+ (is (false? (entity-retrieval.core/available?))))))
+ (testing "fully licensed + pgvector, but no way to compute embeddings -> unavailable"
+ (with-redefs [semantic.db.datasource/db-url "jdbc:postgresql://stub"]
+ (mt/with-premium-features #{:library :library-retrieval}
+ ;; an unrecognized provider hits embedding-supported?'s :default (false)
+ (mt/with-dynamic-fn-redefs [semantic.embedding/get-configured-model (constantly {:provider "no-embedder"})]
+ (is (false? (entity-retrieval.core/available?)))))))))
(deftest ^:sequential entity-retrieval-available?-requires-a-populated-index-test
(testing "the curated tool is offered only once the index has documents (else the nlq profile falls back)"
(when semantic.db.datasource/db-url
- (mt/with-premium-features #{:library :semantic-search}
+ (mt/with-premium-features #{:library :library-retrieval}
(let [suffix (System/nanoTime)
ds (semantic.db.datasource/ensure-initialized-data-source!)]
(mt/with-dynamic-fn-redefs [semantic.embedding/get-configured-model
@@ -163,7 +182,7 @@
(deftest ^:sequential ranks-by-similarity-test
(testing "search ranks library documents by cosine similarity, nearest first"
(when semantic.db.datasource/db-url
- (mt/with-premium-features #{:library :semantic-search}
+ (mt/with-premium-features #{:library :library-retrieval}
(let [suffix (System/nanoTime)
ds (semantic.db.datasource/ensure-initialized-data-source!)
q "the query"]
@@ -206,7 +225,7 @@
;; deterministic, no network) against isolated temp tables, with the reconciler run directly in
;; place of the background Quartz job.
(when semantic.db.datasource/db-url
- (mt/with-premium-features #{:library :semantic-search}
+ (mt/with-premium-features #{:library :library-retrieval}
(let [suffix (System/nanoTime)
ds (semantic.db.datasource/ensure-initialized-data-source!)
q "monthly revenue per region"
@@ -258,7 +277,7 @@
(deftest ^:sequential doc-type-boost-breaks-ties-test
(testing "on a distance tie, a name match outranks a synonym match (blended ORDER BY, not raw NN)"
(when semantic.db.datasource/db-url
- (mt/with-premium-features #{:library :semantic-search}
+ (mt/with-premium-features #{:library :library-retrieval}
(let [suffix (System/nanoTime)
ds (semantic.db.datasource/ensure-initialized-data-source!)
q "the query"
@@ -295,7 +314,7 @@
;; until the next reconcile rebuilds it; search must degrade to [] rather than throw in that window.
(testing "an index/query vector dimension mismatch degrades search to [] instead of throwing"
(when semantic.db.datasource/db-url
- (mt/with-premium-features #{:library :semantic-search}
+ (mt/with-premium-features #{:library :library-retrieval}
(let [suffix (System/nanoTime)
ds (semantic.db.datasource/ensure-initialized-data-source!)]
(mt/with-dynamic-fn-redefs [semantic.embedding/get-configured-model
diff --git a/enterprise/backend/test/metabase_enterprise/entity_retrieval/reconcile_test.clj b/enterprise/backend/test/metabase_enterprise/entity_retrieval/reconcile_test.clj
index ef5403ec07f..4f94d1300c9 100644
--- a/enterprise/backend/test/metabase_enterprise/entity_retrieval/reconcile_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/entity_retrieval/reconcile_test.clj
@@ -103,7 +103,7 @@
(deftest ^:sequential reconcile-lifecycle-test
;; :library lets us publish a Table into a library-data collection; the reconcile enumerates the
;; library tree via collections/library-collection + descendant-ids, so we build a real library root.
- (mt/with-premium-features #{:library :semantic-search}
+ (mt/with-premium-features #{:library :library-retrieval}
(with-isolated-index [ds]
(let [model semantic.tu/mock-embedding-model]
(mt/with-temp [:model/Collection {lib-id :id} {:type "library" :location "/"}
@@ -158,7 +158,7 @@
(deftest ^:sequential reconcile!-runs-to-completion-test
(testing "reconcile! blocks until the run completes and a second run is idempotent"
- (mt/with-premium-features #{:library :semantic-search}
+ (mt/with-premium-features #{:library :library-retrieval}
(with-isolated-index [ds]
(collections.tu/with-library [{data :data}]
(let [model semantic.tu/mock-embedding-model]
@@ -177,7 +177,7 @@
(is (=? {:inserted 0 :deleted 0} (reconcile/reconcile! ds (constantly model))))))))))))
(deftest ^:sequential rebuild-on-model-change-test
- (mt/with-premium-features #{:library :semantic-search}
+ (mt/with-premium-features #{:library :library-retrieval}
(with-isolated-index [ds]
(let [model semantic.tu/mock-embedding-model]
(mt/with-temp [:model/Collection {lib-id :id} {:type "library" :location "/"}
@@ -205,7 +205,7 @@
(deftest ^:sequential measures-and-segments-indexed-and-hydrated-test
(testing "measures/segments on a published library table are indexed and hydrate with parent-table context"
- (mt/with-premium-features #{:library :semantic-search}
+ (mt/with-premium-features #{:library :library-retrieval}
(with-isolated-index [ds]
(let [model semantic.tu/mock-embedding-model
orders (mt/id :orders)
@@ -242,7 +242,7 @@
(deftest ^:sequential failed-insert-spares-only-that-entitys-orphans-test
(testing "a failed insert spares that entity's orphans; an unrelated entity's orphans still GC"
- (mt/with-premium-features #{:library :semantic-search}
+ (mt/with-premium-features #{:library :library-retrieval}
(with-isolated-index [ds]
(let [model semantic.tu/mock-embedding-model]
(mt/with-temp [:model/Collection {lib-id :id} {:type "library" :location "/"}
@@ -267,7 +267,7 @@
(deftest ^:sequential reconcile-entity!-targets-one-slice-test
(testing "reconcile-entity! reconciles only the given entity's docs, leaving other entities untouched"
- (mt/with-premium-features #{:library :semantic-search}
+ (mt/with-premium-features #{:library :library-retrieval}
(with-isolated-index [ds]
(collections.tu/with-library [{data :data}]
(let [model semantic.tu/mock-embedding-model]
@@ -290,7 +290,7 @@
(deftest ^:sequential reconcile-entity!-leaving-library-deletes-all-test
(testing "reconcile-entity! on an entity that has left the library GCs all of its docs"
- (mt/with-premium-features #{:library :semantic-search}
+ (mt/with-premium-features #{:library :library-retrieval}
(with-isolated-index [ds]
(collections.tu/with-library [{data :data}]
(let [model semantic.tu/mock-embedding-model]
@@ -309,7 +309,7 @@
(deftest ^:sequential reconcile-entity!-ai-context-removal-keeps-name-test
(testing "removing an entity's ai_context and reconciling it GCs synonym/example docs but keeps name/description"
- (mt/with-premium-features #{:library :semantic-search}
+ (mt/with-premium-features #{:library :library-retrieval}
(with-isolated-index [ds]
(collections.tu/with-library [{data :data}]
(let [model semantic.tu/mock-embedding-model]
@@ -329,7 +329,7 @@
(deftest ^:sequential reconcile!-keeps-ai-context-across-a-card-type-flip-test
(testing "a full reconcile matches ai_context by entity class, so relabelling a card keeps its synonyms"
- (mt/with-premium-features #{:library :semantic-search}
+ (mt/with-premium-features #{:library :library-retrieval}
(with-isolated-index [ds]
(collections.tu/with-library [{metrics :metrics}]
(let [model semantic.tu/mock-embedding-model]
@@ -351,7 +351,7 @@
(deftest ^:sequential library-entity-matches-library-entities-test
(testing "library-entity (point lookup) agrees with library-entities (full scan) for members and non-members"
- (mt/with-premium-features #{:library :semantic-search}
+ (mt/with-premium-features #{:library :library-retrieval}
(collections.tu/with-library [{data :data metrics :metrics}]
(mt/with-temp [:model/Database {db-id :id} {}
:model/Table {tbl :id} {:db_id db-id :collection_id (:id data) :is_published true
@@ -382,7 +382,7 @@
(deftest ^:sequential reconcile-entity!-on-first-build-repopulates-whole-library-test
(testing "a targeted reconcile that creates the index (first caller, empty table) repopulates the whole library"
- (mt/with-premium-features #{:library :semantic-search}
+ (mt/with-premium-features #{:library :library-retrieval}
(with-isolated-index [ds]
(collections.tu/with-library [{data :data}]
(let [model semantic.tu/mock-embedding-model]
@@ -400,7 +400,7 @@
(deftest ^:sequential reconcile-entity!-on-rebuild-repopulates-whole-library-test
(testing "a targeted reconcile that triggers a model/format rebuild repopulates the whole library, not just the one entity"
- (mt/with-premium-features #{:library :semantic-search}
+ (mt/with-premium-features #{:library :library-retrieval}
(with-isolated-index [ds]
(collections.tu/with-library [{data :data}]
(let [model semantic.tu/mock-embedding-model]
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_test.clj
index 3e4ad35b222..239f788011c 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_test.clj
@@ -373,3 +373,23 @@
(t2/select-one :model/SemanticSearchTokenTracking)]
(is (= :query request_type))
(is (= 13 total_tokens))))))))))))
+
+(deftest embedding-supported?-test
+ (testing "ai-service: supported iff an embedding-service base URL or the ai-service base URL is set"
+ (mt/with-temporary-setting-values [ee-embedding-service-base-url "http://embed"]
+ (is (true? (embedding/embedding-supported? {:provider "ai-service"}))))
+ ;; ai-service-base-url is :enabled? only under the AI-managed / metabot-v3 features, so license one to
+ ;; exercise that branch (and to make the setting settable/restorable here).
+ (mt/with-premium-features #{:metabot-v3}
+ (mt/with-temporary-setting-values [ee-embedding-service-base-url nil ai-service-base-url "http://ai"]
+ (is (true? (embedding/embedding-supported? {:provider "ai-service"})))))
+ (mt/with-temporary-setting-values [ee-embedding-service-base-url nil]
+ (is (false? (embedding/embedding-supported? {:provider "ai-service"})))))
+ (testing "openai: supported iff the API key is set"
+ (mt/with-temporary-setting-values [llm-openai-api-key "sk-test"]
+ (is (true? (embedding/embedding-supported? {:provider "openai"}))))
+ (mt/with-temporary-setting-values [llm-openai-api-key nil]
+ (is (false? (embedding/embedding-supported? {:provider "openai"})))))
+ (testing "ollama is always supported; an unrecognized provider is not (:default)"
+ (is (true? (embedding/embedding-supported? {:provider "ollama"})))
+ (is (false? (embedding/embedding-supported? {:provider "no-embedder"})))))
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/test_util.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/test_util.clj
index 2655afe530b..27061cf11ae 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/test_util.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/test_util.clj
@@ -309,6 +309,7 @@
(defmethod semantic.embedding/get-embedding "mock" [_ text & {:as _opts}] (get-mock-embedding text))
(defmethod semantic.embedding/get-embeddings-batch "mock" [_ texts & {:as _opts}] (get-mock-embeddings-batch texts))
(defmethod semantic.embedding/pull-model "mock" [_])
+(defmethod semantic.embedding/embedding-supported? "mock" [_] true)
#_{:clj-kondo/ignore [:metabase/test-helpers-use-non-thread-safe-functions]}
(defn query-index [search-context]
diff --git a/frontend/src/metabase-types/api/mocks/settings.ts b/frontend/src/metabase-types/api/mocks/settings.ts
index 3bbf32c68f2..eaeb2beafc1 100644
--- a/frontend/src/metabase-types/api/mocks/settings.ts
+++ b/frontend/src/metabase-types/api/mocks/settings.ts
@@ -146,6 +146,7 @@ export const createMockTokenFeatures = (
"transforms-basic": false,
"transforms-python": false,
library: false,
+ library_retrieval: false,
"support-users": false,
tenants: false,
writable_connection: false,
diff --git a/frontend/src/metabase-types/api/settings.ts b/frontend/src/metabase-types/api/settings.ts
index 6d3aade894d..2c21f950f5e 100644
--- a/frontend/src/metabase-types/api/settings.ts
+++ b/frontend/src/metabase-types/api/settings.ts
@@ -376,6 +376,7 @@ export const tokenFeatures = [
"transforms-python",
"transforms-basic",
"library",
+ "library_retrieval",
"support-users",
"tenants",
"writable_connection",
diff --git a/src/metabase/entity_retrieval/mirror.clj b/src/metabase/entity_retrieval/mirror.clj
index 46a43ca0bd6..6664b9d3f11 100644
--- a/src/metabase/entity_retrieval/mirror.clj
+++ b/src/metabase/entity_retrieval/mirror.clj
@@ -4,14 +4,15 @@
The appdb (`osi_ai_context` + library membership) is authoritative; the index carries one embedding
per value-document and serves the `retrieve_library_entities` Metabot tool's similarity search.
These `defenterprise` shims route to [[metabase-enterprise.entity-retrieval.core]] when an enterprise
- license with semantic search is present; otherwise they no-op or return empty."
+ license with library retrieval is present; otherwise they no-op or return empty."
(:require
[metabase.premium-features.core :refer [defenterprise]]))
(defenterprise entity-retrieval-available?
- "Whether the library entity index can serve a query right now: a pgvector store is configured and the
- license includes semantic search. `false` in OSS (no index). Re-evaluated per use — pgvector config is
- boot-fixed but the feature can flip at runtime."
+ "Whether the library entity index can serve a query right now: pgvector configured, the `:library` and
+ `:library-retrieval` features licensed, an embedding backend available, and the index built for the
+ current model and populated. `false` in OSS (no index). Re-evaluated per use — pgvector config is
+ boot-fixed but license/settings can flip at runtime."
metabase-enterprise.entity-retrieval.core
[]
false)
diff --git a/src/metabase/metabot/agent/profiles.clj b/src/metabase/metabot/agent/profiles.clj
index 0e89beb6476..b71d664f07f 100644
--- a/src/metabase/metabot/agent/profiles.clj
+++ b/src/metabase/metabot/agent/profiles.clj
@@ -168,7 +168,7 @@
#'tools/ask-for-sql-clarification-tool]})
;; :nlq and :nlq-fallback are a pair selected by the library index's health (see [[get-profile]]): when the
-;; curated index can serve queries the agent discovers data through retrieve_library_entities; otherwise it
+;; library index can serve queries the agent discovers data through retrieve_library_entities; otherwise it
;; falls back to the general nlq search. They differ only in the discovery tool and the prompt that explains
;; it. The redirect keeps the external profile-id :nlq, so telemetry / recent-views / skills are unaffected.
(register-profile!
@@ -255,7 +255,7 @@
;;; API
(defn- nlq-fallback?
- "Whether a :nlq request should be served the general-search fallback: true when the curated library index
+ "Whether a :nlq request should be served the general-search fallback: true when the library index
can't answer (not configured/licensed, or empty). Keeps data discovery working before the first reconcile
and on OSS / unlicensed instances."
[profile-id]
diff --git a/src/metabase/metabot/tools/entity_retrieval.clj b/src/metabase/metabot/tools/entity_retrieval.clj
index 9e11d810b9b..072e01323ba 100644
--- a/src/metabase/metabot/tools/entity_retrieval.clj
+++ b/src/metabase/metabot/tools/entity_retrieval.clj
@@ -166,14 +166,14 @@
matches))
;; No capability gate: this tool lives only in the :nlq profile, which [[metabase.metabot.agent.profiles/
-;; get-profile]] serves (curated, with this tool) only when entity-retrieval-available? — otherwise it
+;; get-profile]] serves (the library variant, with this tool) only when entity-retrieval-available? — otherwise it
;; redirects to :nlq-fallback (general search). That single probe is the sole gate, so prompt and tools can't
;; disagree about index availability.
(mu/defn ^{:tool-name "retrieve_library_entities"
:scope scope/agent-search}
retrieve-library-entities-tool
- "Find the best data to answer the user's request from the library — the curated set of entities vetted
- for the agent (published tables, library metrics/models, and their measures/segments). Phrase
+ "Find the best data to answer the user's request from the library — the set of entities published to it
+ (published tables, library metrics/models, and their measures/segments). Phrase
`user_search_prompt` as a full natural-language description of the data wanted (it is matched on
meaning, not keywords).
diff --git a/src/metabase/osi/ai_context/api.clj b/src/metabase/osi/ai_context/api.clj
index c93567eaed8..d33b779f2c4 100644
--- a/src/metabase/osi/ai_context/api.clj
+++ b/src/metabase/osi/ai_context/api.clj
@@ -137,12 +137,14 @@
This call never reuses a reconcile already in progress (it may have started before your latest change);
it starts one if the index is idle, otherwise it queues a single follow-up that any other waiting calls
share.
- Requires the semantic search feature; returns a 400 when it isn't configured."
+ Requires the library entity-retrieval feature; returns a 400 when the index is unavailable (the feature
+ isn't licensed, or the pgvector store or embedding backend isn't configured)."
[_route-params
_query-params]
(api/check-superuser)
(api/check-400 (entity-retrieval/force-reconcile!)
- "The library entity index requires semantic search, which is not configured."))
+ (str "The library entity index is unavailable: it needs the library entity-retrieval "
+ "feature plus a configured pgvector store and embedding backend.")))
#_{:clj-kondo/ignore [:metabase/validate-defendpoint-has-response-schema]}
(api.macros/defendpoint :delete "/:entity-type/:entity-local-id"
diff --git a/src/metabase/premium_features/core.clj b/src/metabase/premium_features/core.clj
index f86e25e7c82..33e5fdb0109 100644
--- a/src/metabase/premium_features/core.clj
+++ b/src/metabase/premium_features/core.clj
@@ -62,6 +62,7 @@
enable-database-auth-providers?
enable-database-routing?
enable-library?
+ enable-library-retrieval?
enable-metabase-ai-managed?
enable-metabot-v3?
enable-dependencies?
diff --git a/src/metabase/premium_features/settings.clj b/src/metabase/premium_features/settings.clj
index 19ff2ed0b3f..b8c891f1161 100644
--- a/src/metabase/premium_features/settings.clj
+++ b/src/metabase/premium_features/settings.clj
@@ -276,6 +276,11 @@
"Should we enable the semantic search backend?"
:semantic-search)
+(define-premium-feature ^{:added "0.63.0"} enable-library-retrieval?
+ "Should we enable the Metabot library entity-retrieval tool (retrieve_library_entities)?
+ Independent of `:semantic-search`: this gates only that tool, not the general semantic search engine."
+ :library-retrieval)
+
(define-premium-feature ^{:added "0.57.0"} table-data-editing?
"Should we allow users to edit the data within tables?"
:table-data-editing)
@@ -390,6 +395,7 @@
:database_auth_providers (enable-database-auth-providers?)
:database_routing (enable-database-routing?)
:library (enable-library?)
+ :library_retrieval (enable-library-retrieval?)
:dependencies (enable-dependencies?)
:schema-viewer (enable-schema-viewer?)
:development_mode (development-mode?)
diff --git a/test/metabase/analytics/stats_test.clj b/test/metabase/analytics/stats_test.clj
index 1a84ae54a79..cffe786e70a 100644
--- a/test/metabase/analytics/stats_test.clj
+++ b/test/metabase/analytics/stats_test.clj
@@ -549,6 +549,7 @@
:data-complexity-score
:development-mode
:library
+ :library-retrieval
:embedding
:embedding-sdk
:embedding-simple
diff --git a/test/metabase/metabot/tools/entity_retrieval_test.clj b/test/metabase/metabot/tools/entity_retrieval_test.clj
index 5a311101389..43c05126e81 100644
--- a/test/metabase/metabot/tools/entity_retrieval_test.clj
+++ b/test/metabase/metabot/tools/entity_retrieval_test.clj
@@ -10,7 +10,7 @@
(set! *warn-on-reflection* true)
(deftest tool-shape-oss-fallback-test
- (testing "without the semantic-search feature the tool returns the standard empty result shape"
+ (testing "without the library-retrieval feature the tool returns the standard empty result shape"
;; Pin the feature off so the defenterprise call takes its OSS fallback regardless of any ambient
;; premium token (e.g. an all-features dev token) or rows left in a local pgvector store.
(mt/with-premium-features #{}
diff --git a/test/metabase/metabot/tools_test.clj b/test/metabase/metabot/tools_test.clj
index 9bb6d0393c2..0ca3b94e910 100644
--- a/test/metabase/metabot/tools_test.clj
+++ b/test/metabase/metabot/tools_test.clj
@@ -79,7 +79,7 @@
(is (contains? tools "construct_notebook_query"))
(is (contains? tools "create_chart"))
(is (not (contains? tools "retrieve_library_entities")))))
- ;; Entity retrieval available (pgvector configured + semantic-search licensed): the curated library tool
+ ;; Entity retrieval available (pgvector configured + library-retrieval licensed): the curated library tool
;; replaces general search. Exactly one discovery tool survives capability filtering.
(mt/with-dynamic-fn-redefs [entity-retrieval/entity-retrieval-available? (constantly true)]
(let [tools (tools-for-profile :nlq)]