Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Resolve the default search engine from capability precedence (#77418)

Chris Truter6188ef843f96
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj
index 42adcc8ee17..71d065f447e 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj
@@ -3,7 +3,6 @@
(:require
[clojure.string :as str]
[medley.core :as m]
- [metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource]
[metabase-enterprise.semantic-search.embedders]
[metabase-enterprise.semantic-search.embedding]
[metabase-enterprise.semantic-search.env :as semantic.env]
@@ -17,7 +16,6 @@
[metabase.search.config :as search.config]
[metabase.search.engine :as search.engine]
[metabase.tracing.core :as tracing]
- [metabase.util :as u]
[metabase.util.log :as log]
[potemkin :as p]
[toucan2.realize :as t2.realize]))
@@ -29,30 +27,25 @@
[metabase-enterprise.semantic-search.embedding
get-embeddings-batch])
-(defn- fallback-engine
- "Find the highest priority search engine available for fallback."
- []
- (u/seek #(not= :search.engine/semantic %) (search.engine/supported-engines)))
+(defn- fallback-engine []
+ (search.engine/fallback-engine :search.engine/semantic))
(defn- index-active? [pgvector index-metadata]
(boolean (semantic.index-metadata/get-active-index-state pgvector index-metadata)))
-;; TODO: url should likely reside in settings
(defenterprise supported?
"Enterprise implementation of semantic search engine support check."
:feature :semantic-search
[]
- (and
- (some? semantic.db.datasource/db-url)
- (semantic.settings/semantic-search-enabled)))
+ (semantic.util/semantic-search-available?))
(defn build-hnsw-index-async!
"Build the HNSW index on the active semantic search index in the background, returning promptly.
- No-ops when semantic search isn't available on this instance. Backs the just-in-time HNSW build, which
+ No-ops when semantic search isn't active on this instance. Backs the just-in-time HNSW build, which
runs only when an instance is configured to the `:hnsw` vector-search strategy."
[]
- (when (semantic.util/semantic-search-available?)
+ (when (semantic.util/semantic-search-active?)
(future
(try
(semantic.pgvector-api/ensure-active-hnsw-index! (semantic.env/get-pgvector-datasource!)
@@ -190,7 +183,11 @@
(let [pgvector (semantic.env/get-pgvector-datasource!)
index-metadata (semantic.env/get-index-metadata)]
(if-not (index-active? pgvector index-metadata)
- (log/debug "repair-index! called prior to init!")
+ ;; Semantic can become active at runtime (kill switch re-enabled, or added to additional-search-engines)
+ ;; without init! ever having run; initializing here lets the periodic repair task backfill the index.
+ (do
+ (log/info "No active semantic index, initializing it instead of repairing")
+ (init! searchable-documents {}))
(semantic.repair/with-repair-table!
pgvector
(fn [repair-table-name]
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/index_metadata.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/index_metadata.clj
index b4cf72bcc59..ad3736e86b6 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/index_metadata.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/index_metadata.clj
@@ -204,7 +204,8 @@
(defn- index-table-exists? [pgvector index]
(semantic.util/table-exists? pgvector (:table-name index)))
-(defn- control-and-metadata-tables-exist?
+(defn control-and-metadata-tables-exist?
+ "Have the index bookkeeping tables been created, i.e. has init ever run against this pgvector DB?"
[pgvector index-metadata]
(and (semantic.util/table-exists? pgvector (:metadata-table-name index-metadata))
(semantic.util/table-exists? pgvector (:control-table-name index-metadata))))
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/settings.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/settings.clj
index ea2690b3bf0..6ba4545bec4 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/settings.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/settings.clj
@@ -2,7 +2,6 @@
(:require
[metabase.events.core :as events]
[metabase.llm.settings :as llm-settings]
- [metabase.premium-features.core :as premium-features]
[metabase.search.config :as search.config]
[metabase.settings.core :as setting :refer [defsetting]]
[metabase.util.i18n :refer [deferred-tru]]))
@@ -92,9 +91,6 @@
:export? false
:encryption :no
:default true
- :getter (fn []
- (and (setting/get-value-of-type :boolean :semantic-search-enabled)
- (premium-features/enable-semantic-search?)))
:type :boolean
:doc false)
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/task/index_cleanup.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/task/index_cleanup.clj
index a538edd3706..f382d11680b 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/task/index_cleanup.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/task/index_cleanup.clj
@@ -9,6 +9,7 @@
[honey.sql.helpers :as sql.helpers]
[java-time.api :as t]
[metabase-enterprise.semantic-search.env :as semantic.env]
+ [metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata]
[metabase-enterprise.semantic-search.settings :as semantic.settings]
[metabase-enterprise.semantic-search.util :as semantic.u]
[metabase.task.core :as task]
@@ -165,11 +166,15 @@
(defn- cleanup-stale-indexes-and-gate-tombstones!
[]
(when (semantic.u/semantic-search-available?)
- (let [pgvector (semantic.env/get-pgvector-datasource!)
- index-metadata (semantic.env/get-index-metadata)]
- (cleanup-stale-indexes! pgvector index-metadata)
- (cleanup-old-gate-tombstones! pgvector index-metadata)
- (cleanup-orphan-repair-tables! pgvector))))
+ (let [pgvector (semantic.env/get-pgvector-datasource!)
+ index-metadata (semantic.env/get-index-metadata)]
+ ;; Available but never initialized is a steady state (semantic neither default nor additional);
+ ;; the bookkeeping tables only exist after init, and the cleanup queries would throw without them.
+ ;; Still runs while merely available, so an opted-out instance's leftover indexes get cleaned.
+ (when (semantic.index-metadata/control-and-metadata-tables-exist? pgvector index-metadata)
+ (cleanup-stale-indexes! pgvector index-metadata)
+ (cleanup-old-gate-tombstones! pgvector index-metadata)
+ (cleanup-orphan-repair-tables! pgvector)))))
(def ^:private cleanup-job-key (jobs/key "metabase.task.semantic-index-cleanup.job"))
(def ^:private cleanup-trigger-key (triggers/key "metabase.task.semantic-index-cleanup.trigger"))
@@ -180,7 +185,7 @@
(cleanup-stale-indexes-and-gate-tombstones!))
(defmethod task/init! ::SemanticIndexCleanup [_]
- (when (semantic.u/semantic-search-available?)
+ (when (semantic.u/semantic-search-configured?)
(let [job (jobs/build
(jobs/of-type SemanticIndexCleanup)
(jobs/with-identity cleanup-job-key))
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/task/index_repair.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/task/index_repair.clj
index 01f1097b05e..4553bcfe39f 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/task/index_repair.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/task/index_repair.clj
@@ -20,7 +20,7 @@
(task/defjob ^{DisallowConcurrentExecution true
:doc "Runs repair-index! to maintain semantic search consistency"}
SemanticIndexRepair [_ctx]
- (when (semantic.u/semantic-search-available?)
+ (when (semantic.u/semantic-search-active?)
(log/with-context {:quartz-job-type 'SemanticIndexRepair}
(try
(log/info "Starting semantic search index repair")
@@ -30,7 +30,7 @@
(log/error e "Failed to complete semantic search index repair"))))))
(defmethod task/init! ::SemanticIndexRepair [_]
- (when (semantic.u/semantic-search-available?)
+ (when (semantic.u/semantic-search-configured?)
(let [job (jobs/build
(jobs/of-type SemanticIndexRepair)
(jobs/with-identity repair-job-key))
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/task/indexer.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/task/indexer.clj
index 0045a21b5f5..a0c9c733650 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/task/indexer.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/task/indexer.clj
@@ -5,11 +5,13 @@
[clojurewerkz.quartzite.triggers :as triggers]
[metabase-enterprise.semantic-search.core :as semantic.core]
[metabase-enterprise.semantic-search.env :as semantic.env]
+ [metabase-enterprise.semantic-search.index :as semantic.index]
+ [metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata]
[metabase-enterprise.semantic-search.indexer :as semantic-search.indexer]
[metabase-enterprise.semantic-search.settings :as semantic.settings]
[metabase-enterprise.semantic-search.util :as semantic.u]
[metabase.search.config :as search.config]
- [metabase.search.engine :as search.engine]
+ [metabase.search.core :as search]
[metabase.task.core :as task]
[metabase.util.log :as log])
(:import (java.time Duration Instant)
@@ -29,20 +31,41 @@
;; but if I do that quartz cannot initialize the job - there is probably a way around this.
(defonce ^:private execution-thread-ref (volatile! nil))
+(defn- hnsw-strategy? []
+ (contains? search.config/hnsw-index-backed-strategies
+ (semantic.settings/semantic-search-vector-strategy)))
+
(deftype ^{DisallowConcurrentExecution true
:doc "Runs an indexer process for a time, expects to be rescheduled to continue"}
SemanticSearchIndexer []
org.quartz.Job
(execute [_ _]
- (when (semantic.u/semantic-search-available?)
+ (when (semantic.u/semantic-search-active?)
(log/with-context {:quartz-job-type 'SemanticSearchIndexer}
- (when (search.engine/supported-engine? :search.engine/semantic)
- (try
- (vreset! execution-thread-ref (Thread/currentThread))
- (semantic-search.indexer/quartz-job-run! (semantic.env/get-pgvector-datasource!) (semantic.env/get-index-metadata))
- (finally
- (locking execution-thread-ref
- (vreset! execution-thread-ref nil))))))))
+ (try
+ (vreset! execution-thread-ref (Thread/currentThread))
+ (let [pgvector (semantic.env/get-pgvector-datasource!)
+ index-metadata (semantic.env/get-index-metadata)]
+ (if-let [{:keys [index]} (semantic.index-metadata/get-active-index-state pgvector index-metadata)]
+ (do
+ ;; The strategy setter's build event no-ops while semantic is inactive, so an index-backed
+ ;; strategy configured before (re)activation arrives here with no HNSW index. CONCURRENTLY
+ ;; registers the index in pg_indexes as soon as the build starts, so this fires only once.
+ (when (and (hnsw-strategy?)
+ (not (semantic.u/index-exists? pgvector (semantic.index/hnsw-index-name index))))
+ (semantic.core/build-hnsw-index-async!))
+ (semantic-search.indexer/quartz-job-run! pgvector index-metadata))
+ ;; Engines can activate at runtime (license applied, kill switch re-enabled,
+ ;; additional-search-engines set on another node); initializing from the next tick heals
+ ;; every such path within seconds. Initialize all active engines, not just semantic:
+ ;; activation may also have activated dependencies with no index.
+ (do
+ (search/init-index!)
+ (when (hnsw-strategy?)
+ (semantic.core/build-hnsw-index-async!)))))
+ (finally
+ (locking execution-thread-ref
+ (vreset! execution-thread-ref nil)))))))
org.quartz.InterruptableJob
(interrupt [_]
;; locking required here to avoid racing with the unset in the finally
@@ -55,7 +78,7 @@
(def ^:private ^Duration run-frequency (Duration/parse "PT20S"))
(defmethod task/init! ::SemanticSearchIndexer [_]
- (when (semantic.u/semantic-search-available?)
+ (when (semantic.u/semantic-search-configured?)
(let [job (jobs/build
(jobs/of-type SemanticSearchIndexer)
(jobs/store-durably)
@@ -73,6 +96,5 @@
;; Safety net: an instance that booted already configured for an HNSW-index-backed strategy (set via env
;; var, or set before an active index existed) never saw the setter's transition event, so build the
;; index now. Covers :hnsw and the :hnsw-iterative-* strategies, which all query through the index.
- (when (contains? search.config/hnsw-index-backed-strategies
- (semantic.settings/semantic-search-vector-strategy))
+ (when (hnsw-strategy?)
(semantic.core/build-hnsw-index-async!))))
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/task/metric_collector.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/task/metric_collector.clj
index da7e1006e45..b2e45e054c4 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/task/metric_collector.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/task/metric_collector.clj
@@ -65,7 +65,9 @@
(log/warn "DLQ table does not exist. Index may not have been initialized.")))
(defn- collect-metrics! []
- (when (semantic.u/semantic-search-available?)
+ ;; Active, not merely available: on an available-but-inactive instance the index tables never exist,
+ ;; and the collectors would warn about them on every run.
+ (when (semantic.u/semantic-search-active?)
(let [pgvector (semantic.env/get-pgvector-datasource!)
index-metadata (semantic.env/get-index-metadata)]
(collect-gate-size! pgvector)
@@ -80,7 +82,7 @@
(defmethod task/init! ::SemanticMetricCollector
[_]
- (when (semantic.u/semantic-search-available?)
+ (when (semantic.u/semantic-search-configured?)
(let [job (jobs/build
(jobs/of-type SemanticMetricCollector)
(jobs/with-identity collector-job-key))
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/task/usage_trimmer.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/task/usage_trimmer.clj
index 2b04df57dae..9a8d4a7e9df 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/task/usage_trimmer.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/task/usage_trimmer.clj
@@ -33,7 +33,7 @@
(defmethod task/init! ::SemanticSearchUsageTrimmer
[_]
- (when (semantic.u/semantic-search-available?)
+ (when (semantic.u/semantic-search-configured?)
(let [job (jobs/build
(jobs/of-type SemanticSearchUsageTrimmer)
(jobs/with-identity trimmer-job-key))
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/util.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/util.clj
index 7bf0e5dba45..69fe3590a3b 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/util.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/util.clj
@@ -1,7 +1,9 @@
(ns metabase-enterprise.semantic-search.util
(:require
[metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource]
+ [metabase-enterprise.semantic-search.settings :as semantic.settings]
[metabase.premium-features.core :as premium-features]
+ [metabase.search.engine :as search.engine]
[next.jdbc :as jdbc]
[next.jdbc.result-set :as jdbc.rs]))
@@ -23,8 +25,33 @@
{:builder-fn jdbc.rs/as-unqualified-lower-maps})
(:index_exists false)))
+(defn semantic-search-configured?
+ "Is a pgvector DB configured for this instance?
+ The boot-static input: gates Quartz job scheduling, which happens once at startup.
+ The runtime gates (license, kill switch, engine activity) are checked per job execution instead, so
+ flipping them never requires a restart.
+ This check becomes a DB probe under BOT-1796."
+ []
+ (string? (not-empty semantic.db.datasource/db-url)))
+
+(defn semantic-search-capable?
+ "Does this instance have the infrastructure for semantic search: the premium feature and a pgvector DB.
+ Deliberately excludes the kill switch, which is checked per execution so it works at runtime."
+ []
+ ;; Cheap gate first: semantic-search-configured? becomes a DB probe under BOT-1796.
+ (and (premium-features/has-feature? :semantic-search)
+ (semantic-search-configured?)))
+
(defn semantic-search-available?
- "Predicate to check whether semantic search is available on the instance."
+ "Whether semantic search can run on this instance: capable and not disabled by the kill switch.
+ Engine selection, hygiene tasks, and metrics key off this."
+ []
+ (and (semantic-search-capable?)
+ (semantic.settings/semantic-search-enabled)))
+
+(defn semantic-search-active?
+ "Is the semantic index being maintained on this instance?
+ Anything that writes to the index (or pays for embeddings) must gate on this, not on availability:
+ a supported engine that is neither the default nor an additional engine has no index worth feeding."
[]
- (and (string? (not-empty semantic.db.datasource/db-url))
- (premium-features/has-feature? :semantic-search)))
+ (contains? (set (search.engine/active-engines)) :search.engine/semantic))
diff --git a/enterprise/backend/test/metabase_enterprise/data_studio/search_test.clj b/enterprise/backend/test/metabase_enterprise/data_studio/search_test.clj
index d9e11ff700d..5407b91e2ba 100644
--- a/enterprise/backend/test/metabase_enterprise/data_studio/search_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/data_studio/search_test.clj
@@ -7,9 +7,17 @@
[metabase.permissions.core :as perms]
[metabase.permissions.models.data-permissions :as data-perms]
[metabase.search.core :as search]
+ [metabase.search.engine :as search.engine]
[metabase.test :as mt]
[metabase.util.random :refer [random-name]]))
+(defn- test-engines
+ "The engines to exercise: in-place always, appdb only where the app db supports it.
+ An explicit search_engine the instance cannot serve returns a 400 rather than falling back."
+ []
+ (filter #(search.engine/supported-engine? (keyword "search.engine" %))
+ ["in-place" "appdb"]))
+
(deftest published-table-collection-filter-test
(mt/with-premium-features #{:library}
(testing "Published tables are included in collection-filtered search"
@@ -26,7 +34,7 @@
:model/Table {table-4 :id} {:name "Root Published Table" :is_published true :collection_id nil}]
;; Initialize search index for appdb engine
(search/reindex! {:async? false :in-place? true})
- (doseq [engine ["in-place" "appdb"]]
+ (doseq [engine (test-engines)]
(testing (str "with engine " engine)
(testing "Global search"
(let [results (mt/user-http-request :crowberto :get 200 "search" :search_engine engine)]
@@ -85,7 +93,7 @@
(mt/with-temp
[:model/Table {root-table :id} {:name "Root Published Searchable" :is_published true :collection_id nil}]
(search/reindex! {:async? false :in-place? true})
- (doseq [engine ["in-place" "appdb"]]
+ (doseq [engine (test-engines)]
(testing (str "with engine " engine)
(let [results (mt/user-http-request :crowberto :get 200 "search" :q "Root Published Searchable" :search_engine engine)]
(is (some #(= root-table (:id %)) (:data results))
@@ -104,7 +112,7 @@
:db_id db-id
:is_published false}]
(search/reindex! {:async? false :in-place? true})
- (doseq [engine ["in-place" "appdb"]]
+ (doseq [engine (test-engines)]
(testing (str "with engine " engine)
(let [results (mt/user-http-request :crowberto :get 200 "search"
:q "Context" :models "table" :search_engine engine)
@@ -132,7 +140,7 @@
;; Explicitly deny both view and query perms so this assertion doesn't depend on global defaults.
(data-perms/set-database-permission! (perms/all-users-group) (mt/id) :perms/view-data :blocked)
(data-perms/set-database-permission! (perms/all-users-group) (mt/id) :perms/create-queries :no)
- (doseq [engine ["in-place" "appdb"]]
+ (doseq [engine (test-engines)]
(search/reindex! {:async? false :in-place? true})
(mt/with-non-admin-groups-no-root-collection-perms
(let [results (mt/user-http-request :rasta :get 200 "search"
@@ -157,7 +165,7 @@
;; Grant data permissions on the table
(data-perms/set-database-permission! (perms/all-users-group) (mt/id) :perms/view-data :unrestricted)
(data-perms/set-database-permission! (perms/all-users-group) (mt/id) :perms/create-queries :query-builder)
- (doseq [engine ["in-place" "appdb"]]
+ (doseq [engine (test-engines)]
(search/reindex! {:async? false :in-place? true})
;; Remove collection permissions so the user has data perms but NO collection access
(mt/with-non-admin-groups-no-root-collection-perms
@@ -197,7 +205,7 @@
:model/Card {outside-card :id} {:name (str search-term " Outside Card")
:collection_id outside-coll}]
(search/reindex! {:async? false :in-place? true})
- (doseq [engine ["in-place" "appdb"]]
+ (doseq [engine (test-engines)]
(testing (str "with engine " engine)
(let [results (mt/user-http-request :crowberto :get 200 "search"
:q search-term
@@ -221,7 +229,7 @@
;; Explicitly grant data/query perms so this does not rely on global test defaults.
(data-perms/set-database-permission! (perms/all-users-group) (mt/id) :perms/view-data :unrestricted)
(data-perms/set-database-permission! (perms/all-users-group) (mt/id) :perms/create-queries :query-builder)
- (doseq [engine ["in-place" "appdb"]]
+ (doseq [engine (test-engines)]
(search/reindex! {:async? false :in-place? true})
(mt/with-non-admin-groups-no-root-collection-perms
(let [result-ids (->> (mt/user-http-request :rasta :get 200 "search"
@@ -257,7 +265,7 @@
(perms/grant-collection-read-permissions! (perms/all-users-group) accessible)
(perms/revoke-collection-permissions! (perms/all-users-group) no-access)
(search/reindex! {:async? false :in-place? true})
- (doseq [engine ["in-place" "appdb"]]
+ (doseq [engine (test-engines)]
(testing (str "with engine " engine)
(let [result-ids (->> (mt/user-http-request :rasta :get 200 "search"
:q search-term :search_engine engine)
diff --git a/enterprise/backend/test/metabase_enterprise/metabot/tools/semantic_search_test.clj b/enterprise/backend/test/metabase_enterprise/metabot/tools/semantic_search_test.clj
index 572786b9adb..812f29e6ea0 100644
--- a/enterprise/backend/test/metabase_enterprise/metabot/tools/semantic_search_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/metabot/tools/semantic_search_test.clj
@@ -120,7 +120,7 @@
(mt/with-test-user :rasta
(semantic.tu/with-test-db! {:mode :mock-initialized}
(with-and-without-semantic-search! test-mock-embeddings
- (search.tu/with-new-search-and-legacy-search
+ (search.tu/with-appdb-search-and-legacy-search
(let [semantic-support? (search.engine/supported-engine? :search.engine/semantic)]
;; "belligerent" and "bellicose" are semantically similar to our search terms
;; ("combative", "quarrelsome") but should NOT match since we're only doing keyword search
@@ -146,7 +146,7 @@
(mt/with-test-user :rasta
(semantic.tu/with-test-db! {:mode :mock-initialized}
(with-semantic-search-if-available! test-mock-embeddings
- (search.tu/with-new-search-and-legacy-search
+ (search.tu/with-appdb-search-and-legacy-search
;; "belligerent" and "baseline" will match via keyword search (exact match in term-queries)
;; "ancillary" and "adjunct" will match via semantic search (similar embeddings)
;; "bellicose" and "quixotic" should NOT match (not in search terms)
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/api_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/api_test.clj
index 0863f72e177..133d1e3e221 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/api_test.clj
@@ -56,29 +56,31 @@
(deftest re-init-endpoint-test
(testing "POST /api/search/re-init with semantic search support"
(mt/with-premium-features #{:semantic-search}
- (semantic.tu/with-test-db! {:mode :mock-indexed}
- (let [original-index semantic.tu/mock-index
- original-table-name (:table-name original-index)
- new-index (mt/with-dynamic-fn-redefs [semantic.index/model-table-suffix (constantly 345)]
- (#'semantic.pgvector-api/fresh-index semantic.tu/mock-index-metadata semantic.tu/mock-embedding-model :force-reset? true))
- new-table-name (:table-name new-index)
- pgvector (semantic.env/get-pgvector-datasource!)]
- (is (semantic.tu/table-exists-in-db? original-table-name))
- (is (not (semantic.tu/table-exists-in-db? new-table-name)))
- (let [best-index (semantic.index-metadata/find-compatible-index! pgvector semantic.tu/mock-index-metadata semantic.tu/mock-embedding-model)]
- (is (=? original-index (:index best-index)))
- (is (:active best-index)))
- (testing "re-init creates the new index"
- (mt/with-dynamic-fn-redefs [semantic.index/model-table-suffix (constantly 345)]
- (let [response (mt/user-http-request :crowberto :post 200 "search/re-init")]
- (is (contains? response :message))))
- (is (not= original-table-name new-table-name))
+ ;; Semantic is not the default engine here, so it must be activated explicitly.
+ (mt/with-temporary-setting-values [additional-search-engines ["semantic"]]
+ (semantic.tu/with-test-db! {:mode :mock-indexed}
+ (let [original-index semantic.tu/mock-index
+ original-table-name (:table-name original-index)
+ new-index (mt/with-dynamic-fn-redefs [semantic.index/model-table-suffix (constantly 345)]
+ (#'semantic.pgvector-api/fresh-index semantic.tu/mock-index-metadata semantic.tu/mock-embedding-model :force-reset? true))
+ new-table-name (:table-name new-index)
+ pgvector (semantic.env/get-pgvector-datasource!)]
(is (semantic.tu/table-exists-in-db? original-table-name))
- (is (semantic.tu/table-exists-in-db? new-table-name))
- (is (zero? (semantic.tu/index-count new-index))))
- (let [best-index (semantic.index-metadata/find-compatible-index! pgvector semantic.tu/mock-index-metadata semantic.tu/mock-embedding-model)]
- (is (=? new-index (:index best-index)))
- (is (:active best-index)))
- (testing "Index can be populated after re-init"
- (semantic.tu/upsert-index! (semantic.tu/mock-documents) :index new-index)
- (is (pos? (semantic.tu/index-count new-index)))))))))
+ (is (not (semantic.tu/table-exists-in-db? new-table-name)))
+ (let [best-index (semantic.index-metadata/find-compatible-index! pgvector semantic.tu/mock-index-metadata semantic.tu/mock-embedding-model)]
+ (is (=? original-index (:index best-index)))
+ (is (:active best-index)))
+ (testing "re-init creates the new index"
+ (mt/with-dynamic-fn-redefs [semantic.index/model-table-suffix (constantly 345)]
+ (let [response (mt/user-http-request :crowberto :post 200 "search/re-init")]
+ (is (contains? response :message))))
+ (is (not= original-table-name new-table-name))
+ (is (semantic.tu/table-exists-in-db? original-table-name))
+ (is (semantic.tu/table-exists-in-db? new-table-name))
+ (is (zero? (semantic.tu/index-count new-index))))
+ (let [best-index (semantic.index-metadata/find-compatible-index! pgvector semantic.tu/mock-index-metadata semantic.tu/mock-embedding-model)]
+ (is (=? new-index (:index best-index)))
+ (is (:active best-index)))
+ (testing "Index can be populated after re-init"
+ (semantic.tu/upsert-index! (semantic.tu/mock-documents) :index new-index)
+ (is (pos? (semantic.tu/index-count new-index))))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/repair_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/repair_test.clj
index b3d5fff638d..4616911649a 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/repair_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/repair_test.clj
@@ -5,7 +5,10 @@
[honey.sql :as sql]
[java-time.api :as t]
[metabase-enterprise.semantic-search.core :as semantic.core]
+ [metabase-enterprise.semantic-search.embedding :as semantic.embedding]
[metabase-enterprise.semantic-search.env :as semantic.env]
+ [metabase-enterprise.semantic-search.index :as semantic.index]
+ [metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata]
[metabase-enterprise.semantic-search.repair :as semantic.repair]
[metabase-enterprise.semantic-search.test-util :as semantic.tu]
[metabase-enterprise.semantic-search.util :as semantic.util]
@@ -87,6 +90,25 @@
(is (tombstone? deleted-card-entry) "Deleted card should be a tombstone in gate table")
(is (tombstone? deleted-dashboard-entry) "Deleted dashboard should be a tombstone in gate table"))))))))))
+(deftest repair-index-initializes-when-missing-test
+ (testing "repair-index! initializes the index when none is active, so runtime activation backfills"
+ (mt/with-premium-features #{:semantic-search}
+ (semantic.tu/with-test-db! {:mode :blank}
+ ;; with-redefs mirrors the :mock-initialized bindings, which include non-fn values that
+ ;; with-dynamic-fn-redefs cannot proxy.
+ #_{:clj-kondo/ignore [:metabase/prefer-with-dynamic-fn-redefs]}
+ (with-redefs [semantic.embedding/get-configured-model (fn [] semantic.tu/mock-embedding-model)
+ semantic.index-metadata/default-index-metadata semantic.tu/mock-index-metadata
+ semantic.index/model-table-suffix semantic.tu/mock-table-suffix]
+ (let [pgvector (semantic.env/get-pgvector-datasource!)
+ index-metadata (semantic.env/get-index-metadata)]
+ (is (nil? (semantic.index-metadata/get-active-index-state pgvector index-metadata)))
+ (semantic.core/repair-index! [(create-test-document "card" 1 "Dog Training Guide")])
+ (is (some? (semantic.index-metadata/get-active-index-state pgvector index-metadata)))
+ (testing "the supplied documents are gated for backfill"
+ (let [gate-contents (gate-table-contents pgvector (:gate-table-name index-metadata))]
+ (is (some? (:document (gate-entry-by-id gate-contents "card_1"))))))))))))
+
(deftest repair-table-cleanup-test
(testing "The repair table gets cleaned up properly at the end of a repair-index! job"
(mt/with-premium-features #{:semantic-search}
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/task/indexer_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/task/indexer_test.clj
index 62b623b8d0c..b9aa772128a 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/task/indexer_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/task/indexer_test.clj
@@ -16,7 +16,7 @@
;; env var), where the setter's transition event never fired -- see the safety net in sut/init!. Redefs
;; avoid scheduling a real quartz job or spawning a real build future.
(let [builds (atom 0)]
- (mt/with-dynamic-fn-redefs [semantic.u/semantic-search-available? (constantly true)
+ (mt/with-dynamic-fn-redefs [semantic.u/semantic-search-configured? (constantly true)
task/schedule-task! (fn [& _] nil)
semantic.core/build-hnsw-index-async! (fn [] (swap! builds inc))]
(testing ":brute-force does not trigger a build"
diff --git a/enterprise/backend/test/metabase_enterprise/tenants/search_test.clj b/enterprise/backend/test/metabase_enterprise/tenants/search_test.clj
index e82f4665832..5a2506b52a2 100644
--- a/enterprise/backend/test/metabase_enterprise/tenants/search_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/tenants/search_test.clj
@@ -14,7 +14,7 @@
(use-fixtures :once (fixtures/initialize :db))
(use-fixtures :each (fn [thunk] (binding [search.ingestion/*force-sync* true]
- (search.tu/with-new-search-if-available-otherwise-legacy (thunk)))))
+ (search.tu/with-appdb-search-if-available-otherwise-legacy (thunk)))))
(deftest dedicated-tenant-collection-dataset-search-test
(testing "Search returns datasets (models) from dedicated tenant collections"
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj
index d9d7cfd7534..c0783a79d09 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj
@@ -154,7 +154,7 @@
:name query-name)
:model/Transform {python-id :id} (assoc (python-transform-map (str "target_" (u/generate-nano-id)))
:name python-name)]
- (search.tu/with-new-search-and-legacy-search
+ (search.tu/with-appdb-search-and-legacy-search
(testing "no transforms feature"
(mt/with-premium-features #{}
(is (empty? (search-transform-ids search-term)))))
@@ -175,7 +175,7 @@
:name query-name)
:model/Transform {python-id :id} (assoc (python-transform-map (str "target_" (u/generate-nano-id)))
:name python-name)]
- (search.tu/with-new-search-and-legacy-search
+ (search.tu/with-appdb-search-and-legacy-search
(mt/with-premium-features #{:transforms-basic :transforms-python :hosting}
(is (= #{query-id python-id} (search-transform-ids search-term))))
(mt/with-premium-features #{:transforms-basic :hosting}
@@ -191,7 +191,7 @@
query-name (str search-term "-query")]
(mt/with-temp [:model/Transform {query-id :id} (assoc (query-transform-payload (str "target_" (u/generate-nano-id)))
:name query-name)]
- (search.tu/with-new-search-and-legacy-search
+ (search.tu/with-appdb-search-and-legacy-search
(let [ids (search-api-transform-ids :crowberto search-term)]
(is (empty? ids))
(is (not (contains? ids query-id)))))))))))
diff --git a/src/metabase/metabot/self/features.clj b/src/metabase/metabot/self/features.clj
index b5a48a21ceb..ee5a22b5178 100644
--- a/src/metabase/metabot/self/features.clj
+++ b/src/metabase/metabot/self/features.clj
@@ -9,7 +9,9 @@
(def feature-predicates
"Registry of feature keywords to predicate functions."
- {:semantic-search #(search.engine/supported-engine? :search.engine/semantic)})
+ ;; Status, not capability: advertising semantic_queries in tool schemas when no semantic index is
+ ;; maintained makes the LLM emit semantic queries that would run as keyword searches.
+ {:semantic-search #(= :ok (search.engine/engine-status :search.engine/semantic))})
(defn feature-available?
"Check if a feature is available. Unknown features return true (fail-open)."
diff --git a/src/metabase/metabot/tools/search.clj b/src/metabase/metabot/tools/search.clj
index 6f833587179..f73b113e58b 100644
--- a/src/metabase/metabot/tools/search.clj
+++ b/src/metabase/metabot/tools/search.clj
@@ -307,7 +307,7 @@
semantic? #{:search.engine/semantic}
semantic-engine (u/seek semantic? (search.engine/active-engines))
fallback-engine (when semantic-engine
- (u/seek (comp not semantic?) (search.engine/supported-engines)))
+ (search.engine/fallback-engine semantic-engine))
fused-results (if semantic-engine
;; Perform semantic and non-semantic search respectively, then fuse results.
(reciprocal-rank-fusion
diff --git a/src/metabase/search/api.clj b/src/metabase/search/api.clj
index cae6faa6021..f7d21b50c92 100644
--- a/src/metabase/search/api.clj
+++ b/src/metabase/search/api.clj
@@ -11,6 +11,7 @@
[metabase.request.core :as request]
[metabase.search.config :as search.config]
[metabase.search.core :as search]
+ [metabase.search.engine :as search.engine]
[metabase.search.ingestion :as ingestion]
[metabase.search.settings :as search.settings]
[metabase.search.task.search-index :as task.search-index]
@@ -26,7 +27,8 @@
(set! *warn-on-reflection* true)
-(def ^:private engine-cookie-name "metabase.SEARCH_ENGINE")
+;; Renamed from metabase.SEARCH_ENGINE so long-lived cookies written before engine validation existed stay inert.
+(def ^:private engine-cookie-name "metabase.SEARCH_ENGINE_OVERRIDE")
(defn- cookie-expiry []
;; 20 years should be long enough to trial an experimental search engine
@@ -42,6 +44,16 @@
:path "/"
:expires (cookie-expiry)}))))
+(defn- clear-engine-cookie! [respond]
+ (fn [response]
+ (respond
+ (response/set-cookie response
+ engine-cookie-name
+ ""
+ {:http-only true
+ :path "/"
+ :max-age 0}))))
+
(defn- process-non-temporal-dim-ids
"Parse and process non-temporal dimension IDs JSON string.
Filters out null values and sorts ascending, returning as JSON string."
@@ -57,15 +69,65 @@
(log/warn "Failed to parse non-temporal dimension IDs:" (ex-message e))
nil))))
+(defn- param->engine
+ "Parse a search_engine param or cookie value into an engine keyword, nil when blank.
+ Malformed non-blank values parse to an unknown engine rather than nil, so explicit requests 400."
+ [value]
+ (when-not (str/blank? value)
+ (search.engine/canonical-engine value)))
+
+(defn- check-engine-serves!
+ "400 when an explicitly requested engine cannot serve searches, naming the cause."
+ [engine]
+ (case (search.engine/engine-status engine)
+ :unknown
+ (throw (ex-info (tru "Unknown search engine: {0}" (name engine)) {:status-code 400}))
+ :unsupported
+ (throw (ex-info (tru "Search engine {0} is not supported on this instance" (name engine)) {:status-code 400}))
+ :inactive
+ (throw (ex-info (tru "Search engine {0} is not enabled; add it to additional-search-engines to use it"
+ (name engine))
+ {:status-code 400}))
+ :ok nil))
+
+(defn- cookie-engine
+ "The engine cookie's engine name when it can still serve, else nil.
+ Cookies outlive configuration, so a stale value degrades to the default instead of erroring."
+ [request]
+ (when-let [engine (param->engine (get-in request [:cookies engine-cookie-name :value]))]
+ (when (= :ok (search.engine/engine-status engine))
+ (name engine))))
+
(defn- +engine-cookie [handler]
(open-api/handler-with-open-api-spec
(fn [request respond raise]
- (if-let [new-engine (get-in request [:params :search_engine])]
- (handler request (set-engine-cookie! respond new-engine) raise)
- (handler (->> (get-in request [:cookies engine-cookie-name :value])
- (assoc-in request [:params :search_engine]))
- respond
- raise)))
+ ;; Endpoints read :query-params (string keys), so that is where the engine must be injected.
+ (let [raw (get-in request [:query-params "search_engine"])]
+ (cond
+ ;; A repeated query param parses as a vector: pass it through for schema validation to reject.
+ (and raw (not (string? raw)))
+ (handler request respond raise)
+
+ ;; An explicit blank unpins: delete the engine cookie and resolve the default.
+ (and raw (str/blank? raw))
+ (handler (assoc-in request [:query-params "search_engine"] nil)
+ (clear-engine-cookie! respond)
+ raise)
+
+ :else
+ (if-let [engine (param->engine raw)]
+ (try
+ (check-engine-serves! engine)
+ (handler (assoc-in request [:query-params "search_engine"] (name engine))
+ (set-engine-cookie! respond (name engine))
+ raise)
+ (catch Exception e
+ (raise e)))
+ (let [cookie (cookie-engine request)]
+ (handler (cond-> request
+ cookie (assoc-in [:query-params "search_engine"] cookie))
+ respond
+ raise))))))
(fn [prefix]
(open-api/open-api-spec handler prefix))))
diff --git a/src/metabase/search/appdb/core.clj b/src/metabase/search/appdb/core.clj
index fd02e9c96f3..1669c7eb7ae 100644
--- a/src/metabase/search/appdb/core.clj
+++ b/src/metabase/search/appdb/core.clj
@@ -16,7 +16,6 @@
[metabase.search.impl :as search.impl]
[metabase.search.ingestion :as search.ingestion]
[metabase.search.permissions :as search.permissions]
- [metabase.search.settings :as search.settings]
[metabase.search.spec :as search.spec]
[metabase.search.util :as search.util]
[metabase.settings.core :as setting]
@@ -45,13 +44,7 @@
#{:postgres :h2})
(defmethod search.engine/supported-engine? :search.engine/appdb [_]
- (and (or config/is-dev?
- ;; TODO (Chris 2025-11-07) This backwards dependency is unfortunate, we should find a better solution.
- ;; Perhaps just an explicit setting for enabling it.
- ;; This also opens us up to swapping out the fallback, e.g. to elastic search.
- ;; if the default engine is semantic we want appdb to be available, as we want to mix results
- (#{"appdb" "semantic"} (some-> (search.settings/search-engine) name)))
- (supported-db? (mdb/db-type))))
+ (supported-db? (mdb/db-type)))
(defmethod search.engine/disjunction :search.engine/appdb [_ terms]
(when (seq terms)
diff --git a/src/metabase/search/core.clj b/src/metabase/search/core.clj
index d173c740573..879be246b09 100644
--- a/src/metabase/search/core.clj
+++ b/src/metabase/search/core.clj
@@ -84,7 +84,10 @@
(defmethod analytics.core/initial-value :metabase-search/engine-active
[_ {:keys [engine]}]
- (if (search.engine/supported-engine? (keyword "search.engine" engine)) 1 0))
+ ;; Can the engine serve queries: in-place always can, indexed engines only while their index is maintained.
+ (if (= :ok (search.engine/engine-status (keyword "search.engine" engine)))
+ 1
+ 0))
(defn supports-index?
"Does this instance support a search index, of any sort?"
@@ -94,7 +97,8 @@
(defn init-index!
"Ensure there is an index ready to be populated."
[& {:as opts}]
- (when (supports-index?)
+ (search.engine/log-resolution!)
+ (when-let [engines (seq (search.engine/active-engines))]
(log/info "Initializing search indexes")
(tracing/with-span :search "search.init-index" {}
(lib-be/with-metadata-provider-cache
@@ -103,7 +107,7 @@
(let [timer (u/start-timer)
report (reduce (partial merge-with max)
nil
- (for [e (search.engine/active-engines)]
+ (for [e engines]
(search.engine/init! e opts)))
duration (u/since-ms timer)]
(if (seq report)
@@ -120,7 +124,7 @@
(throw e)))))))
(defn- reindex-logic! [opts]
- (when (supports-index?)
+ (when-let [engines (seq (search.engine/active-engines))]
(tracing/with-span :search "search.reindex" {}
(lib-be/with-metadata-provider-cache
(try
@@ -128,7 +132,7 @@
(let [timer (u/start-timer)
report (reduce (partial merge-with max)
nil
- (for [e (search.engine/active-engines)]
+ (for [e engines]
(search.engine/reindex! e opts)))
duration (u/since-ms timer)]
(analytics/inc! :metabase-search/index-reindex-ms duration)
diff --git a/src/metabase/search/engine.clj b/src/metabase/search/engine.clj
index 001f5d6e8dc..068dc5d600b 100644
--- a/src/metabase/search/engine.clj
+++ b/src/metabase/search/engine.clj
@@ -1,16 +1,22 @@
;; The interface encapsulating the various search engine backends.
(ns metabase.search.engine
(:require
- [metabase.search.settings :as settings]))
+ [clojure.string :as str]
+ [metabase.search.settings :as settings]
+ [metabase.util :as u]
+ [metabase.util.log :as log]))
(def ^:private default-engine-precedence
- "In the absence of explicit configuration, these are the engines to try using, in decreasing order of preference."
+ "The engines to use as the default, in decreasing order of preference.
+ The first supported one wins, unless overridden by the [[settings/search-engine]] setting."
[:search.engine/semantic
:search.engine/appdb
:search.engine/in-place])
(defmulti supported-engine?
- "Does this instance support the given engine?"
+ "Can this instance run the given engine?
+ Implementations must be pure capability checks: app-db type, premium features, required infrastructure.
+ They must not read [[settings/search-engine]], which selects among the supported engines."
{:arglists '([engine])}
identity)
@@ -18,6 +24,14 @@
(throw (ex-info (format "Unknown search engine: %s" engine)
{:engine engine})))
+(defmulti dependencies
+ "Engines whose indexes this engine needs in order to serve queries.
+ Semantic, for example, mixes appdb results into its own and falls back to appdb when its index is unavailable."
+ {:arglists '([engine])}
+ identity)
+
+(defmethod dependencies :default [_] nil)
+
(defmulti results
"Return a reducible of the search result matching a given query."
{:arglists '([search-context])}
@@ -97,25 +111,71 @@
;; If we end up with more "abstract" nodes, we may want a better way to filter them out.
(keys (dissoc (methods supported-engine?) :default)))
+(defn known-engine?
+ "Is the engine, or an engine it derives from, registered?"
+ [engine]
+ (let [registered? #(contains? (methods supported-engine?) %)]
+ (boolean (some registered? (cons engine (ancestors engine))))))
+
+(def ^:private warned-engine-values
+ "Values already warned about, so misconfiguration warns once rather than on every call."
+ (atom #{}))
+
+(defn- warn-once! [value message]
+ (when-not (contains? @warned-engine-values value)
+ (swap! warned-engine-values conj value)
+ (log/warn message)))
+
+(def ^:private legacy-engine-names
+ "Renamed engines that may still arrive via old configuration or long-lived cookies."
+ {"fulltext" "appdb"})
+
+(defn canonical-engine
+ "Coerce an engine name (string or keyword) to its canonical engine keyword.
+ Resolves legacy names and tolerates the qualified search.engine/ prefix the API echoes in responses.
+ Every configuration and request boundary must canonicalize through this, so that engine identities
+ compare with =.
+ Malformed values yield an unknown engine named after the raw value, never an empty keyword."
+ [value]
+ (let [raw (name value)
+ stripped (str/replace raw #"^search\.engine/" "")
+ engine-name (if (str/blank? stripped) raw stripped)]
+ (keyword "search.engine" (get legacy-engine-names engine-name engine-name))))
+
+(defn- validated-engine
+ "Coerce a configured engine name to a known engine keyword.
+ Unknown values return nil with a one-time warning, so a typo in MB_SEARCH_ENGINE cannot break search."
+ [value]
+ (when value
+ (let [engine (canonical-engine value)]
+ (if (known-engine? engine)
+ engine
+ (warn-once! value (format "Ignoring unknown search engine: %s" value))))))
+
+(defn- configured-engine []
+ ;; The raw setting value, not the public getter: [[settings/search-engine]] resolves its default via
+ ;; [[default-engine]], so reading it here would recurse.
+ (validated-engine (settings/configured-search-engine)))
+
(defn supported-engines
"List the search engines that are supported, in order of usage preference.
The configured engine comes first, if it is supported."
[]
- (let [configured-engine (some->> (settings/search-engine) name (keyword "search.engine"))
- potential-engines (cond->> default-engine-precedence configured-engine (cons configured-engine))]
- (distinct (filter supported-engine? potential-engines))))
-
-(defn active-engines
- "A list of supported search engines for which we will maintain an index, in order of usage preference.
- Excludes :search.engine/in-place, which does not use an index."
+ (let [configured (configured-engine)
+ potential-engines (cond->> default-engine-precedence
+ configured (cons configured))]
+ ;; Drop engines that are not yet registered: the search-engine setting can be read before
+ ;; metabase.search.init loads the engine implementations, and supported-engine? throws on unknown engines.
+ (distinct (filter supported-engine? (filter known-engine? potential-engines)))))
+
+(defn- additional-engines
+ "The supported engines force-enabled by [[settings/additional-search-engines]]."
[]
- (remove #{:search.engine/in-place} (supported-engines)))
-
-(defn known-engine?
- "Is the given engine recognized?"
- [engine]
- (let [registered? #(contains? (methods supported-engine?) %)]
- (some registered? (cons engine (ancestors engine)))))
+ (->> (settings/additional-search-engines)
+ (map str/trim)
+ (remove str/blank?)
+ (keep validated-engine)
+ (filter supported-engine?)))
(defmulti disjunction
"Given multiple terms to search for, reduce this to a search expression that matches any of them in a single search.
@@ -124,10 +184,51 @@
(fn [search-engine _terms] search-engine))
(defn default-engine
- "In the absence of an explicit engine argument in a request, which engine should be used?"
+ "The engine that serves requests with no explicit engine argument.
+ The configured override when supported, otherwise the first supported engine in the precedence."
+ []
+ (let [configured (configured-engine)
+ default (first (supported-engines))]
+ (when (and configured (not= configured default))
+ (warn-once! [:unsupported configured]
+ (format "Configured search engine %s is not supported on this instance, using %s instead"
+ configured default)))
+ default))
+
+(defn log-resolution!
+ "Log the engine that serves search."
[]
- ;; TODO (Chris 2025-11-07) It would be good to have a warning on start up whenever this is *not* what's configured.
- (first (supported-engines)))
+ (log/infof "Search will be served by the %s engine" (default-engine)))
+
+(defn active-engines
+ "The engines for which we maintain an index, default engine first.
+ The default engine and any [[settings/additional-search-engines]], plus their supported [[dependencies]].
+ Excludes :search.engine/in-place, which does not use an index."
+ []
+ (if-let [default (default-engine)]
+ (->> (concat [default] (additional-engines))
+ (mapcat #(cons % (filter supported-engine? (filter known-engine? (dependencies %)))))
+ distinct
+ (remove #{:search.engine/in-place}))
+ []))
+
+(defn engine-status
+ "Whether the engine can serve queries, and if not, why: :ok, :unknown, :unsupported, or :inactive.
+ In-place needs no index, so it serves whenever supported; other engines must be in [[active-engines]].
+ Membership is exact: callers must resolve engine aliases before asking."
+ [engine]
+ (cond
+ (not (known-engine? engine)) :unknown
+ (not (supported-engine? engine)) :unsupported
+ (or (= engine :search.engine/in-place)
+ (contains? (set (active-engines)) engine)) :ok
+ :else :inactive))
+
+(defn fallback-engine
+ "The engine to mix results with and fall back to when `engine` cannot serve alone.
+ Prefers engines with a maintained index over merely supported ones."
+ [engine]
+ (u/seek #(not= engine %) (concat (active-engines) (supported-engines))))
(defmethod disjunction :default [_ terms] terms)
diff --git a/src/metabase/search/semantic/core.clj b/src/metabase/search/semantic/core.clj
index e3a4dd2c5c0..c4225889d69 100644
--- a/src/metabase/search/semantic/core.clj
+++ b/src/metabase/search/semantic/core.clj
@@ -67,6 +67,9 @@
(log/warn e "Semantic search engine not supported")
false)))
+(defmethod search.engine/dependencies :search.engine/semantic [_]
+ [:search.engine/appdb])
+
(defmethod search.engine/results :search.engine/semantic
[search-ctx]
(results search-ctx))
diff --git a/src/metabase/search/settings.clj b/src/metabase/search/settings.clj
index 5af7155ad69..48051731a9c 100644
--- a/src/metabase/search/settings.clj
+++ b/src/metabase/search/settings.clj
@@ -1,8 +1,10 @@
(ns metabase.search.settings
(:require
+ [metabase.app-db.core :as mdb]
[metabase.appearance.core :as appearance]
[metabase.settings.core :as setting :refer [defsetting]]
- [metabase.util.i18n :as i18n]))
+ [metabase.util.i18n :as i18n]
+ [metabase.util.log :as log]))
(defsetting search-typeahead-enabled
(i18n/deferred-tru "Enable typeahead search in the {0} navbar?"
@@ -13,15 +15,44 @@
:export? true
:audit :getter)
+(defn configured-search-engine
+ "The operator's engine override, or nil when unset.
+ [[search-engine]] resolves a default when unset; use this to see what was explicitly configured."
+ []
+ (setting/get-value-of-type :keyword :search-engine))
+
(defsetting search-engine
- (i18n/deferred-tru "Which engine to use by default for search. Supported values are :in-place, :appdb, and :semantic")
+ (i18n/deferred-tru "The engine that serves search. Supported values are :in-place, :appdb, and :semantic. When unset, the best supported engine is used.")
:visibility :authenticated
:export? false
:setter :none
- ;; TODO (Chris 2025-11-07) Would be good to remove the default and just use [search.engine/default-engine-precedence]
- :default :appdb
+ ;; Reading resolves the engine that actually serves search; [[configured-search-engine]] is the raw override.
+ :getter (fn []
+ (some-> ((requiring-resolve 'metabase.search.engine/default-engine)) name keyword))
:type :keyword)
+(defsetting additional-search-engines
+ (i18n/deferred-tru "Engines to keep active (indexed and queryable per-request) in addition to the default engine.")
+ :visibility :internal
+ :export? false
+ :encryption :no
+ :default nil
+ :type :csv
+ ;; Post-commit + future: the trigger check must observe the final committed settings state (a surrounding
+ ;; set-many! may include the setting that makes the engine supported) and stay off the transaction's thread.
+ :setter (fn [new-value]
+ (let [before (set ((requiring-resolve 'metabase.search.engine/active-engines)))
+ result (setting/set-value-of-type! :csv :additional-search-engines new-value)]
+ (mdb/do-after-commit
+ #(future
+ (try
+ ((requiring-resolve 'metabase.search.task.search-index/trigger-init-for-newly-active-engines!)
+ before)
+ (catch Throwable t
+ (log/error t "Failed to trigger search index init for newly active engines")))))
+ result))
+ :doc false)
+
(defsetting experimental-search-weight-overrides
(i18n/deferred-tru "Used to override weights used for search ranking")
:visibility :internal
diff --git a/src/metabase/search/task/search_index.clj b/src/metabase/search/task/search_index.clj
index bedbfda5697..696d5559601 100644
--- a/src/metabase/search/task/search_index.clj
+++ b/src/metabase/search/task/search_index.clj
@@ -5,6 +5,7 @@
[clojurewerkz.quartzite.triggers :as triggers]
[metabase.app-db.cluster-lock :as cluster-lock]
[metabase.search.core :as search]
+ [metabase.search.engine :as search.engine]
[metabase.search.ingestion :as ingestion]
[metabase.startup.core :as startup]
[metabase.task.core :as task]
@@ -45,6 +46,11 @@
(cluster-lock/with-cluster-lock cluster-lock-name
(search/reindex! {:async? false})))
+(task/defjob ^{DisallowConcurrentExecution true
+ :doc "Ensure indexes exist for the active engines"}
+ SearchIndexInit [_ctx]
+ (init!))
+
;; Atom holding a promise that is delivered when the background init thread finishes.
;; nil when no init has been started — [[wait-for-init!]] returns immediately in that case.
(defonce ^:private init-promise (atom nil))
@@ -65,6 +71,24 @@
(deliver p true)))))
.start)))
+(defn trigger-init-for-newly-active-engines!
+ "Trigger the init job if any engine beyond `before` is now active.
+ A newly activated engine needs its index initialized before it can serve; already-active engines are
+ skipped because semantic init re-gates every searchable document.
+ No-op when the init job is not registered, i.e. the scheduler is not running (tests)."
+ [before]
+ (when (and (seq (remove before (search.engine/active-engines)))
+ (task/job-exists? init-job-key))
+ (task/trigger-now! init-job-key)))
+
+;; A durable job with no schedule: triggered on demand when an engine is activated at runtime,
+;; e.g. by the additional-search-engines setter.
+(defmethod task/init! ::SearchIndexInit [_]
+ (task/add-job! (jobs/build
+ (jobs/of-type SearchIndexInit)
+ (jobs/store-durably)
+ (jobs/with-identity init-job-key))))
+
(defmethod task/init! ::SearchIndexReindex [_]
(let [job (jobs/build
(jobs/of-type SearchIndexReindex)
diff --git a/test/metabase/agent_api/api_test.clj b/test/metabase/agent_api/api_test.clj
index c7188ee92da..2d6aeebfa6d 100644
--- a/test/metabase/agent_api/api_test.clj
+++ b/test/metabase/agent_api/api_test.clj
@@ -97,7 +97,7 @@
(deftest search-test
(binding [search.ingestion/*force-sync* true]
- (search.tu/with-new-search-if-available-otherwise-legacy
+ (search.tu/with-appdb-search-if-available-otherwise-legacy
(mt/with-temp [:model/Table _ {:name "AgentSearchTestTable"}]
(testing "Returns search results for term queries"
(is (=? {:data [{:type "table" :name "AgentSearchTestTable"}]
@@ -108,7 +108,7 @@
(deftest search-content-types-test
(testing "search surfaces saved questions, dashboards, and collections (not just tables/metrics/models)"
(binding [search.ingestion/*force-sync* true]
- (search.tu/with-new-search-if-available-otherwise-legacy
+ (search.tu/with-appdb-search-if-available-otherwise-legacy
(mt/with-temp [:model/Card _ {:name "AgentSearchAcmeQuestion"}
:model/Dashboard _ {:name "AgentSearchAcmeDashboard"}
:model/Collection _ {:name "AgentSearchAcmeCollection"}]
@@ -556,7 +556,7 @@
(deftest search-finds-metrics-test
(binding [search.ingestion/*force-sync* true]
- (search.tu/with-new-search-if-available-otherwise-legacy
+ (search.tu/with-appdb-search-if-available-otherwise-legacy
(mt/with-temp [:model/Card _metric {:name "AgentSearchTestMetric"
:type :metric
:database_id (mt/id)
@@ -569,7 +569,7 @@
(deftest search-finds-models-test
(binding [search.ingestion/*force-sync* true]
- (search.tu/with-new-search-if-available-otherwise-legacy
+ (search.tu/with-appdb-search-if-available-otherwise-legacy
(mt/with-temp [:model/Card _model {:name "AgentSearchTestModel"
:type :model
:database_id (mt/id)
diff --git a/test/metabase/analytics/prometheus_test.clj b/test/metabase/analytics/prometheus_test.clj
index ac7118ab845..3b689466183 100644
--- a/test/metabase/analytics/prometheus_test.clj
+++ b/test/metabase/analytics/prometheus_test.clj
@@ -6,7 +6,7 @@
[clojure.test :refer :all]
[iapetos.registry :as registry]
[metabase.analytics.prometheus :as prometheus]
- [metabase.search.core :as search]
+ [metabase.search.engine :as search.engine]
[metabase.test :as mt]
[metabase.test.fixtures :as fixtures]
[metabase.util :as u])
@@ -219,16 +219,17 @@
sum (fn [metric] (reduce + 0 (vals (engine->value metric))))]
(testing "A consistent set of engines is enumerated"
(is (= (engines :metabase-search/engine-active)
- (engines :metabase-search/engine-active))))
+ (engines :metabase-search/engine-default))))
(testing "The values are boolean"
(is (set/superset? #{0 1} (set (vals (engine->value :metabase-search/engine-active)))))
(is (set/superset? #{0 1} (set (vals (engine->value :metabase-search/engine-default))))))
- (testing "Legacy search is always active"
+ (testing "The default engine is active"
+ (is (= 1 (value :metabase-search/engine-active (search.engine/default-engine)))))
+ (testing "In-place can always serve, so it is always active"
(is (= 1 (value :metabase-search/engine-active :in-place))))
- (testing "There is at least one other active engine iff we support an index."
- (if (search/supports-index?)
- (is (< 1 (sum :metabase-search/engine-active)))
- (is (= 1 (sum :metabase-search/engine-active)))))
+ (testing "Beyond in-place, engines are active iff their index is maintained"
+ (is (= (inc (count (remove #{:search.engine/in-place} (search.engine/active-engines))))
+ (sum :metabase-search/engine-active))))
(testing "There is only one default"
(is (= 1 (sum :metabase-search/engine-default))))))
diff --git a/test/metabase/documents/search_test.clj b/test/metabase/documents/search_test.clj
index 0b2a92f1335..cfabdceadd4 100644
--- a/test/metabase/documents/search_test.clj
+++ b/test/metabase/documents/search_test.clj
@@ -133,7 +133,7 @@
;; the legacy in-place engine LIKE-matches the raw prose-mirror JSON.
(mt/with-temp [:model/Document {doc-id :id} {:name "Annual Summary"
:document (documents.test-util/text->prose-mirror-ast "quarterly revenue projections and growth")}]
- (search.tu/with-new-search-and-legacy-search
+ (search.tu/with-appdb-search-and-legacy-search
(testing "found by a term that appears only in the body"
(let [results (mt/user-http-request :crowberto :get 200 "search" :q "projections" :models "document")]
(is (contains? (set (map :id (:data results))) doc-id))))
@@ -153,7 +153,7 @@
:content [{:type "text" :text "see"}
{:type "smartLink"
:attrs {:model "card" :entityId "abc" :label "Customer Retention"}}]}]}}]
- (search.tu/with-new-search-and-legacy-search
+ (search.tu/with-appdb-search-and-legacy-search
(testing "found by a term that appears only in the smart-link label"
(let [results (mt/user-http-request :crowberto :get 200 "search" :q "Retention" :models "document")]
(is (contains? (set (map :id (:data results))) doc-id))))))))
diff --git a/test/metabase/metabot/curation_test.clj b/test/metabase/metabot/curation_test.clj
index aa3e8649fee..daa9a1b93d6 100644
--- a/test/metabase/metabot/curation_test.clj
+++ b/test/metabase/metabot/curation_test.clj
@@ -11,7 +11,7 @@
(deftest curated-ids-consistent-with-index-test
(testing "curated-ids reads the same verdict from source-of-truth tables that the search index
precomputes from the specs, so the two definitions stay consistent (BOT-1570)"
- (search.tu/with-new-search-if-available-without-fallback
+ (search.tu/with-appdb-search-if-available-without-fallback
(mt/with-temp [:model/Database {db :id} {}
:model/Collection {off :id} {:authority_level "official"}
;; name avoids the "curationfixture" search token so the collection itself isn't a
diff --git a/test/metabase/search/api_test.clj b/test/metabase/search/api_test.clj
index 1c07d28b79e..ca0d2fc64df 100644
--- a/test/metabase/search/api_test.clj
+++ b/test/metabase/search/api_test.clj
@@ -17,10 +17,12 @@
[metabase.permissions.core :as perms]
[metabase.permissions.util :as perms-util]
[metabase.revisions.models.revision :as revision]
+ [metabase.search.api :as search.api]
[metabase.search.appdb.core :as search.engines.appdb]
[metabase.search.appdb.index :as search.index]
[metabase.search.config :as search.config]
[metabase.search.core :as search]
+ [metabase.search.engine :as search.engine]
[metabase.search.ingestion :as search.ingestion]
[metabase.search.test-util :as search.tu]
[metabase.test :as mt]
@@ -40,7 +42,7 @@
(def ^:private default-collection {:id false :name nil :authority_level nil :type nil})
(use-fixtures :each (fn [thunk] (binding [search.ingestion/*force-sync* true]
- (search.tu/with-new-search-if-available-otherwise-legacy (thunk)))))
+ (search.tu/with-appdb-search-if-available-otherwise-legacy (thunk)))))
(def ^:private default-search-row
{:archived false
@@ -334,6 +336,85 @@
(is (=? {:engine string?}
(mt/user-http-request :crowberto :get 200 "search" :q "x" :context "search-app"))))))
+(deftest explicit-engine-validation-test
+ (testing "an explicit search_engine that cannot serve returns a 400 naming the reason"
+ (testing "unknown engine"
+ (is (= "Unknown search engine: elastic"
+ (mt/user-http-request :crowberto :get 400 "search" :q "x" :search_engine "elastic"))))
+ (testing "known engine that this instance does not support"
+ (is (= "Search engine semantic is not supported on this instance"
+ (mt/user-http-request :crowberto :get 400 "search" :q "x" :search_engine "semantic"))))
+ (testing "a malformed engine value is rejected, not treated as absent"
+ (is (= "Unknown search engine: search.engine/"
+ (mt/user-http-request :crowberto :get 400 "search" :q "x" :search_engine "search.engine/"))))
+ (when (search.engine/supported-engine? :search.engine/appdb)
+ (testing "supported engine that is neither the default nor an additional engine"
+ ;; Checked below the HTTP layer: this namespace's :each fixture pins the default engine to appdb,
+ ;; so no engine can be made inactive in a real request here.
+ (mt/with-dynamic-fn-redefs [search.engine/active-engines (constantly [])]
+ (is (thrown-with-msg? Exception
+ #"not enabled"
+ (#'search.api/check-engine-serves! :search.engine/appdb)))))))
+ (testing "a servable explicit engine is accepted"
+ (is (=? {:engine "search.engine/in-place"}
+ (mt/user-http-request :crowberto :get 200 "search" :q "x" :search_engine "in-place"))))
+ (testing "a blank engine is treated as absent"
+ (is (=? {:engine string?}
+ (mt/user-http-request :crowberto :get 200 "search" :q "x" :search_engine ""))))
+ (testing "the fully qualified form the API returns in :engine is accepted"
+ (is (=? {:engine "search.engine/in-place"}
+ (mt/user-http-request :crowberto :get 200 "search" :q "x" :search_engine "search.engine/in-place"))))
+ (testing "a repeated search_engine param fails validation rather than erroring"
+ (is (=? {:errors {:search_engine some?}}
+ (mt/user-http-request :crowberto :get 400 "search" :q "x" :search_engine ["appdb" "in-place"])))))
+
+(def ^:private engine-cookie-name @#'search.api/engine-cookie-name)
+
+(deftest engine-cookie-staleness-test
+ (testing "a cookie engine that can still serve is honored"
+ ;; in-place serves on every app db
+ (is (= "in-place" (#'search.api/cookie-engine {:cookies {engine-cookie-name {:value "in-place"}}}))))
+ (testing "a cookie engine that cannot serve degrades to the default"
+ ;; appdb is either unsupported on this app db, or supported with no maintained index
+ (mt/with-dynamic-fn-redefs [search.engine/active-engines (constantly [])]
+ (is (nil? (#'search.api/cookie-engine {:cookies {engine-cookie-name {:value "appdb"}}})))))
+ (testing "an unknown cookie engine degrades to the default"
+ (is (nil? (#'search.api/cookie-engine {:cookies {engine-cookie-name {:value "wut"}}})))))
+
+(deftest engine-cookie-request-test
+ (let [pinned {:request-options {:cookies {engine-cookie-name {:value "in-place"}}}}]
+ (testing "a pinned engine cookie selects the engine"
+ (is (=? {:engine "search.engine/in-place"}
+ (mt/user-http-request :crowberto :get 200 "search" pinned :q "x"))))
+ (testing "an explicit blank search_engine unpins: the cookie is ignored and the default serves"
+ (is (=? {:engine (u/qualified-name (search.engine/default-engine))}
+ (mt/user-http-request :crowberto :get 200 "search" pinned :q "x" :search_engine ""))))))
+
+(deftest engine-cookie-round-trip-test
+ ;; The in-process client is required here: it runs the handler on the test thread, inside the fixture's
+ ;; engine and index bindings.
+ (let [cookie-pattern (re-pattern (str (java.util.regex.Pattern/quote engine-cookie-name) "=([^;]*)"))
+ engine-cookie (fn [response]
+ (some #(second (re-find cookie-pattern %))
+ (u/one-or-many (get-in response [:headers "Set-Cookie"]))))]
+ (testing "an explicit engine pins: the response sets the engine cookie"
+ (let [response (mt/user-http-request-full-response :crowberto :get 200 "search" :q "x" :search_engine "in-place")]
+ (is (= "in-place" (engine-cookie response)))
+ (is (=? {:engine "search.engine/in-place"} (:body response)))))
+ (testing "a request carrying the cookie serves the pinned engine"
+ (is (=? {:engine "search.engine/in-place"}
+ (mt/user-http-request :crowberto :get 200 "search"
+ {:request-options {:cookies {engine-cookie-name {:value "in-place"}}}}
+ :q "x"))))
+ (testing "a blank explicit engine unpins: the response deletes the cookie and the default serves"
+ (let [response (mt/user-http-request-full-response :crowberto :get 200 "search"
+ {:request-options {:cookies {engine-cookie-name {:value "in-place"}}}}
+ :q "x" :search_engine "")]
+ (is (= "" (engine-cookie response)))
+ (is (str/includes? (str/join " " (u/one-or-many (get-in response [:headers "Set-Cookie"])))
+ "Max-Age=0"))
+ (is (=? {:engine (u/qualified-name (search.engine/default-engine))} (:body response)))))))
+
(deftest vector-search-knobs-test
(testing "the vector-search tuning/diagnostic knobs are admin-only"
(doseq [[param value] {:vector_search_ef_search 100
@@ -436,18 +517,14 @@
(let [resp (search-request :crowberto :q "test" :search_engine "appdb" :limit 1)]
;; The index is not populated here, so there's not much interesting to assert.
(is (= "search.engine/appdb" (:engine resp))))))
- (testing "It can use the old search engine name, e.g. for old cookies"
+ (testing "It normalizes the old search engine name, e.g. for old cookies"
(search/init-index! {:force-reset? false :re-populate? false})
(with-search-items-in-root-collection "test"
(let [resp (search-request :crowberto :q "test" :search_engine "fulltext" :limit 1)]
- (is (= "search.engine/fulltext" (:engine resp))))))
- (testing "It will not use an unknown search engine"
- (search/init-index! {:force-reset? false :re-populate? false})
- (with-search-items-in-root-collection "test"
- (let [resp (search-request :crowberto :q "test" :search_engine "wut" :limit 1)]
- (is (#{"search.engine/in-place"
- "search.engine/appdb"}
- (:engine resp))))))))
+ (is (= "search.engine/appdb" (:engine resp))))))
+ (testing "It rejects an unknown search engine"
+ (is (= "Unknown search engine: wut"
+ (mt/user-http-request :crowberto :get 400 "search" :q "test" :search_engine "wut" :limit 1))))))
(defn- get-available-models [& args]
(disj
diff --git a/test/metabase/search/engine_test.clj b/test/metabase/search/engine_test.clj
new file mode 100644
index 00000000000..cc620a740cf
--- /dev/null
+++ b/test/metabase/search/engine_test.clj
@@ -0,0 +1,114 @@
+(ns metabase.search.engine-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase.search.engine :as search.engine]
+ ;; Loaded for side effects: registers the engine implementations.
+ [metabase.search.init]
+ [metabase.search.settings :as search.settings]
+ [metabase.search.task.search-index :as task.search-index]
+ [metabase.task.core :as task]
+ [metabase.test :as mt]
+ [metabase.test.util :as tu]))
+
+(def ^:private all-engines
+ #{:search.engine/semantic :search.engine/appdb :search.engine/in-place})
+
+(defmacro ^:private with-engines
+ "Run `body` with the given engine configuration, bypassing the real capability checks and settings.
+ `supported` is the set of capable engines, `configured` the search-engine setting value, and `additional`
+ the additional-search-engines setting value."
+ [{:keys [supported configured additional]} & body]
+ ;; with-redefs is required: supported-engine? is a multimethod, which with-dynamic-fn-redefs cannot proxy.
+ #_{:clj-kondo/ignore [:metabase/prefer-with-dynamic-fn-redefs]}
+ `(with-redefs [search.engine/supported-engine? ~supported
+ search.engine/known-engine? all-engines
+ search.settings/configured-search-engine (constantly ~configured)
+ search.settings/additional-search-engines (constantly ~additional)]
+ ~@body))
+
+(deftest default-engine-resolution-test
+ (testing "the first supported engine in the precedence wins"
+ (with-engines {:supported all-engines}
+ (is (= :search.engine/semantic (search.engine/default-engine))))
+ (with-engines {:supported #{:search.engine/appdb :search.engine/in-place}}
+ (is (= :search.engine/appdb (search.engine/default-engine))))
+ (with-engines {:supported #{:search.engine/in-place}}
+ (is (= :search.engine/in-place (search.engine/default-engine)))))
+ (testing "a configured engine overrides the precedence"
+ (with-engines {:supported all-engines :configured :appdb}
+ (is (= :search.engine/appdb (search.engine/default-engine))))
+ (with-engines {:supported all-engines :configured :in-place}
+ (is (= :search.engine/in-place (search.engine/default-engine)))))
+ (testing "a configured engine that is not supported falls back to the precedence"
+ (with-engines {:supported #{:search.engine/appdb :search.engine/in-place} :configured :semantic}
+ (is (= :search.engine/appdb (search.engine/default-engine)))))
+ (testing "an unknown configured engine is ignored rather than breaking resolution"
+ (with-engines {:supported all-engines :configured :elastic}
+ (is (= :search.engine/semantic (search.engine/default-engine)))))
+ (testing "a legacy engine name is canonicalized"
+ (with-engines {:supported all-engines :configured :fulltext}
+ (is (= :search.engine/appdb (search.engine/default-engine))))))
+
+(deftest search-engine-setting-test
+ (testing "the setting computes the resolved engine when no value is configured"
+ (with-engines {:supported #{:search.engine/appdb :search.engine/in-place}}
+ (is (= :appdb (search.settings/search-engine))))
+ (with-engines {:supported all-engines}
+ (is (= :semantic (search.settings/search-engine)))))
+ (testing "a configured value is returned as-is"
+ (with-engines {:supported all-engines :configured :in-place}
+ (is (= :in-place (search.settings/search-engine)))))
+ (testing "an override that cannot be honored reports the engine actually serving"
+ (with-engines {:supported #{:search.engine/appdb :search.engine/in-place} :configured :semantic}
+ (is (= :appdb (search.settings/search-engine))))))
+
+(deftest supported-engines-test
+ (testing "the configured engine leads, followed by the precedence"
+ (with-engines {:supported all-engines :configured :in-place}
+ (is (= [:search.engine/in-place :search.engine/semantic :search.engine/appdb]
+ (search.engine/supported-engines))))))
+
+(deftest active-engines-test
+ (testing "a semantic default activates appdb, its dependency"
+ (with-engines {:supported all-engines}
+ (is (= [:search.engine/semantic :search.engine/appdb] (search.engine/active-engines)))))
+ (testing "an appdb default activates only appdb"
+ (with-engines {:supported all-engines :configured :appdb}
+ (is (= [:search.engine/appdb] (search.engine/active-engines)))))
+ (testing "an in-place default activates nothing"
+ (with-engines {:supported #{:search.engine/in-place}}
+ (is (= [] (search.engine/active-engines)))))
+ (testing "additional engines are activated alongside the default, with their dependencies"
+ (with-engines {:supported all-engines :configured :appdb :additional ["semantic"]}
+ (is (= [:search.engine/appdb :search.engine/semantic] (search.engine/active-engines))))
+ (with-engines {:supported all-engines :configured :in-place :additional ["semantic"]}
+ (is (= [:search.engine/semantic :search.engine/appdb] (search.engine/active-engines)))))
+ (testing "unknown or unsupported additional engines are ignored"
+ (with-engines {:supported #{:search.engine/appdb :search.engine/in-place}
+ :configured :appdb
+ :additional ["semantic" "elastic"]}
+ (is (= [:search.engine/appdb] (search.engine/active-engines)))))
+ (testing "legacy engine names in additional engines are canonicalized"
+ (with-engines {:supported all-engines :configured :in-place :additional ["fulltext"]}
+ (is (= [:search.engine/appdb] (search.engine/active-engines)))))
+ (testing "additional engines tolerate csv whitespace, blanks, and qualified names"
+ (with-engines {:supported all-engines :configured :appdb :additional [" semantic" " " "search.engine/appdb"]}
+ (is (= [:search.engine/appdb :search.engine/semantic] (search.engine/active-engines))))))
+
+(deftest additional-search-engines-setter-test
+ (let [triggered (atom [])]
+ (mt/with-dynamic-fn-redefs [task/job-exists? (constantly true)
+ task/trigger-now! (fn [k] (swap! triggered conj k))
+ ;; Track the setting so the setter sees the newly added engine.
+ search.engine/active-engines (fn []
+ (map #(keyword "search.engine" %)
+ (search.settings/additional-search-engines)))]
+ (testing "activating an engine triggers the search index init task"
+ (mt/with-temporary-setting-values [additional-search-engines ["semantic"]]
+ ;; The setter triggers from a post-commit future.
+ (is (tu/poll-until 10000 (some #{task.search-index/init-job-key} @triggered)))
+ (testing "but nothing triggers when no engine is newly active"
+ (reset! triggered [])
+ ;; Deterministic negative: call the trigger check directly with the current engines as the baseline.
+ (task.search-index/trigger-init-for-newly-active-engines! (set (search.engine/active-engines)))
+ (is (empty? @triggered))))))))
diff --git a/test/metabase/search/filter_test.clj b/test/metabase/search/filter_test.clj
index 8d529cd4feb..87e4ccd8adf 100644
--- a/test/metabase/search/filter_test.clj
+++ b/test/metabase/search/filter_test.clj
@@ -14,7 +14,7 @@
(use-fixtures :once (fixtures/initialize :db))
(use-fixtures :each (fn [thunk] (binding [search.ingestion/*force-sync* true]
- (search.tu/with-new-search-if-available-otherwise-legacy (thunk)))))
+ (search.tu/with-appdb-search-if-available-otherwise-legacy (thunk)))))
(defn- filter-keys []
(remove #{:ids} (map :context-key (vals search.config/filters))))
diff --git a/test/metabase/search/ingestion_test.clj b/test/metabase/search/ingestion_test.clj
index 3f2ed8b55d4..776172aaae7 100644
--- a/test/metabase/search/ingestion_test.clj
+++ b/test/metabase/search/ingestion_test.clj
@@ -206,7 +206,7 @@
(deftest curation-signals-surfaced-in-results-test
(testing "curated (all models) and table data_layer ride through legacy_input to appdb search results,
so Metabot can render them (BOT-1570) — not just stored in the filtering column"
- (search.tu/with-new-search-if-available-without-fallback
+ (search.tu/with-appdb-search-if-available-without-fallback
(mt/with-temp [:model/Database {db-id :id} {}
:model/Table _ {:db_id db-id
:name "Curatedgoldtbl"
diff --git a/test/metabase/search/test_util.clj b/test/metabase/search/test_util.clj
index 1fbff5d6b2e..7e1d6f69a1b 100644
--- a/test/metabase/search/test_util.clj
+++ b/test/metabase/search/test_util.clj
@@ -31,43 +31,54 @@
~@body))))
#_{:clj-kondo/ignore [:metabase/test-helpers-use-non-thread-safe-functions]}
-(defmacro with-new-search-if-available*
+(defmacro with-appdb-search-if-available*
"Create a temporary index table for the duration of the body."
[& body]
- `(mt/with-dynamic-fn-redefs [search.engine/default-engine (constantly :search.engine/appdb)]
+ `(mt/with-dynamic-fn-redefs [search.engine/default-engine (constantly :search.engine/appdb)
+ ;; active-engines derives from the default engine, so pinning the default
+ ;; would deactivate semantic; keep it active (and indexed) when supported.
+ search.engine/active-engines (fn []
+ (cond-> [:search.engine/appdb]
+ (search.engine/supported-engine? :search.engine/semantic)
+ (conj :search.engine/semantic)))]
(with-temp-index-table
(search/reindex! {:async? false :in-place? true})
~@body)))
#_{:clj-kondo/ignore [:metabase/test-helpers-use-non-thread-safe-functions]}
-(defmacro with-new-search-if-available-otherwise-legacy
+(defmacro with-appdb-search-if-available-otherwise-legacy
"Create a temporary index table for the duration of the body."
[& body]
`(if (search/supports-index?)
- (with-new-search-if-available* ~@body)
+ (with-appdb-search-if-available* ~@body)
~@body))
-(defmacro with-new-search-if-available-without-fallback
+(defmacro with-appdb-search-if-available-without-fallback
"Create a temporary index table for the duration of the body.
Only runs if the appdb search engine is supported."
[& body]
`(when (search.engine/supported-engine? :search.engine/appdb)
- (with-new-search-if-available* ~@body)))
+ (with-appdb-search-if-available* ~@body)))
(defmacro with-legacy-search
"Ensure legacy search, which doesn't require an index, is used.
Semantic queries go to :search.engine/semantic and keyword queries fall back to :search.engine/in-place."
[& body]
`(mt/with-dynamic-fn-redefs [search.engine/default-engine (constantly :search.engine/in-place)
- search.engine/supported-engines (constantly [:search.engine/semantic :search.engine/in-place])]
+ search.engine/supported-engines (constantly [:search.engine/semantic :search.engine/in-place])
+ ;; active-engines derives from the default engine; semantic must stay active
+ ;; when supported for semantic queries to reach it.
+ search.engine/active-engines (fn []
+ (when (search.engine/supported-engine? :search.engine/semantic)
+ [:search.engine/semantic]))]
~@body))
-(defmacro with-new-search-and-legacy-search
+(defmacro with-appdb-search-and-legacy-search
"Run the body twice, once with the legacy search engine, and once with the appdb search engine."
[& body]
`(do
(with-legacy-search ~@body)
- (with-new-search-if-available-without-fallback ~@body)))
+ (with-appdb-search-if-available-without-fallback ~@body)))
(defmacro with-index-disabled
"Skip any index maintenance during this test."
diff --git a/test/metabase/transforms_rest/api/transform_test.clj b/test/metabase/transforms_rest/api/transform_test.clj
index 7d0230ff9f0..6926a9887ff 100644
--- a/test/metabase/transforms_rest/api/transform_test.clj
+++ b/test/metabase/transforms_rest/api/transform_test.clj
@@ -1876,7 +1876,7 @@
:schema (get-test-schema)
:name (str "target_" (u/generate-nano-id))
:database (mt/id)}}]
- (search.tu/with-new-search-and-legacy-search
+ (search.tu/with-appdb-search-and-legacy-search
(let [transform-ids (search-transform-ids search-term)]
(is (contains? transform-ids query-id))
(is (not (contains? transform-ids python-id)))))))))))
@@ -1894,7 +1894,7 @@
:target {:type "table"
:schema (get-test-schema)
:name (str "target_" (u/generate-nano-id))}}]
- (search.tu/with-new-search-and-legacy-search
+ (search.tu/with-appdb-search-and-legacy-search
(let [results (mt/user-http-request :rasta :get 200 "search" :q search-term :models "transform")
ids (set (map :id (:data results)))]
(is (empty? ids))
@@ -1921,7 +1921,7 @@
:target {:type "table"
:schema (get-test-schema)
:name (str "target_" (u/generate-nano-id))}}]
- (search.tu/with-new-search-and-legacy-search
+ (search.tu/with-appdb-search-and-legacy-search
(is (= #{native-id mbql-id}
(search-transform-ids search-term))))))))))