The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Resolve the default search engine from capability precedence (#77418)
Chris Truter6188ef843f96
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/core.cljindex 42adcc8ee17..71d065f447e 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/core.clj@@ -3,7 +3,6 @@(:require[clojure.string :as str][medley.core :as m]- [metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource][metabase-enterprise.semantic-search.embedders][metabase-enterprise.semantic-search.embedding][metabase-enterprise.semantic-search.env :as semantic.env]@@ -17,7 +16,6 @@[metabase.search.config :as search.config][metabase.search.engine :as search.engine][metabase.tracing.core :as tracing]- [metabase.util :as u][metabase.util.log :as log][potemkin :as p][toucan2.realize :as t2.realize]))@@ -29,30 +27,25 @@[metabase-enterprise.semantic-search.embeddingget-embeddings-batch])-(defn- fallback-engine- "Find the highest priority search engine available for fallback."- []- (u/seek #(not= :search.engine/semantic %) (search.engine/supported-engines)))+(defn- fallback-engine []+ (search.engine/fallback-engine :search.engine/semantic))(defn- index-active? [pgvector index-metadata](boolean (semantic.index-metadata/get-active-index-state pgvector index-metadata)))-;; TODO: url should likely reside in settings(defenterprise supported?"Enterprise implementation of semantic search engine support check.":feature :semantic-search[]- (and- (some? semantic.db.datasource/db-url)- (semantic.settings/semantic-search-enabled)))+ (semantic.util/semantic-search-available?))(defn build-hnsw-index-async!"Build the HNSW index on the active semantic search index in the background, returning promptly.- No-ops when semantic search isn't available on this instance. Backs the just-in-time HNSW build, which+ No-ops when semantic search isn't active on this instance. Backs the just-in-time HNSW build, whichruns only when an instance is configured to the `:hnsw` vector-search strategy."[]- (when (semantic.util/semantic-search-available?)+ (when (semantic.util/semantic-search-active?)(future(try(semantic.pgvector-api/ensure-active-hnsw-index! (semantic.env/get-pgvector-datasource!)@@ -190,7 +183,11 @@(let [pgvector (semantic.env/get-pgvector-datasource!)index-metadata (semantic.env/get-index-metadata)](if-not (index-active? pgvector index-metadata)- (log/debug "repair-index! called prior to init!")+ ;; Semantic can become active at runtime (kill switch re-enabled, or added to additional-search-engines)+ ;; without init! ever having run; initializing here lets the periodic repair task backfill the index.+ (do+ (log/info "No active semantic index, initializing it instead of repairing")+ (init! searchable-documents {}))(semantic.repair/with-repair-table!pgvector(fn [repair-table-name]diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/index_metadata.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/index_metadata.cljindex b4cf72bcc59..ad3736e86b6 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/index_metadata.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/index_metadata.clj@@ -204,7 +204,8 @@(defn- index-table-exists? [pgvector index](semantic.util/table-exists? pgvector (:table-name index)))-(defn- control-and-metadata-tables-exist?+(defn control-and-metadata-tables-exist?+ "Have the index bookkeeping tables been created, i.e. has init ever run against this pgvector DB?"[pgvector index-metadata](and (semantic.util/table-exists? pgvector (:metadata-table-name index-metadata))(semantic.util/table-exists? pgvector (:control-table-name index-metadata))))diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/settings.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/settings.cljindex ea2690b3bf0..6ba4545bec4 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/settings.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/settings.clj@@ -2,7 +2,6 @@(:require[metabase.events.core :as events][metabase.llm.settings :as llm-settings]- [metabase.premium-features.core :as premium-features][metabase.search.config :as search.config][metabase.settings.core :as setting :refer [defsetting]][metabase.util.i18n :refer [deferred-tru]]))@@ -92,9 +91,6 @@:export? false:encryption :no:default true- :getter (fn []- (and (setting/get-value-of-type :boolean :semantic-search-enabled)- (premium-features/enable-semantic-search?))):type :boolean:doc false)diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/task/index_cleanup.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/task/index_cleanup.cljindex a538edd3706..f382d11680b 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/task/index_cleanup.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/task/index_cleanup.clj@@ -9,6 +9,7 @@[honey.sql.helpers :as sql.helpers][java-time.api :as t][metabase-enterprise.semantic-search.env :as semantic.env]+ [metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata][metabase-enterprise.semantic-search.settings :as semantic.settings][metabase-enterprise.semantic-search.util :as semantic.u][metabase.task.core :as task]@@ -165,11 +166,15 @@(defn- cleanup-stale-indexes-and-gate-tombstones- (let [pgvector (semantic.env/get-pgvector-datasource!)- index-metadata (semantic.env/get-index-metadata)]- (cleanup-stale-indexes! pgvector index-metadata)- (cleanup-old-gate-tombstones! pgvector index-metadata)- (cleanup-orphan-repair-tables! pgvector))))+ (let [pgvector (semantic.env/get-pgvector-datasource!)+ index-metadata (semantic.env/get-index-metadata)]+ ;; Available but never initialized is a steady state (semantic neither default nor additional);+ ;; the bookkeeping tables only exist after init, and the cleanup queries would throw without them.+ ;; Still runs while merely available, so an opted-out instance's leftover indexes get cleaned.+ (when (semantic.index-metadata/control-and-metadata-tables-exist? pgvector index-metadata)+ (cleanup-stale-indexes! pgvector index-metadata)+ (cleanup-old-gate-tombstones! pgvector index-metadata)+ (cleanup-orphan-repair-tables! pgvector)))))(def ^:private cleanup-job-key (jobs/key "metabase.task.semantic-index-cleanup.job"))(def ^:private cleanup-trigger-key (triggers/key "metabase.task.semantic-index-cleanup.trigger"))@@ -180,7 +185,7 @@(cleanup-stale-indexes-and-gate-tombstones!))(defmethod task/init! ::SemanticIndexCleanup [_]- (when (semantic.u/semantic-search-available?)+ (when (semantic.u/semantic-search-configured?)(let [job (jobs/build(jobs/of-type SemanticIndexCleanup)(jobs/with-identity cleanup-job-key))diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/task/index_repair.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/task/index_repair.cljindex 01f1097b05e..4553bcfe39f 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/task/index_repair.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/task/index_repair.clj@@ -20,7 +20,7 @@(task/defjob ^{DisallowConcurrentExecution true:doc "Runs repair-index! to maintain semantic search consistency"}SemanticIndexRepair [_ctx]- (when (semantic.u/semantic-search-available?)+ (when (semantic.u/semantic-search-active?)(log/with-context {:quartz-job-type 'SemanticIndexRepair}(try(log/info "Starting semantic search index repair")@@ -30,7 +30,7 @@(log/error e "Failed to complete semantic search index repair"))))))(defmethod task/init! ::SemanticIndexRepair [_]- (when (semantic.u/semantic-search-available?)+ (when (semantic.u/semantic-search-configured?)(let [job (jobs/build(jobs/of-type SemanticIndexRepair)(jobs/with-identity repair-job-key))diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/task/indexer.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/task/indexer.cljindex 0045a21b5f5..a0c9c733650 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/task/indexer.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/task/indexer.clj@@ -5,11 +5,13 @@[clojurewerkz.quartzite.triggers :as triggers][metabase-enterprise.semantic-search.core :as semantic.core][metabase-enterprise.semantic-search.env :as semantic.env]+ [metabase-enterprise.semantic-search.index :as semantic.index]+ [metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata][metabase-enterprise.semantic-search.indexer :as semantic-search.indexer][metabase-enterprise.semantic-search.settings :as semantic.settings][metabase-enterprise.semantic-search.util :as semantic.u][metabase.search.config :as search.config]- [metabase.search.engine :as search.engine]+ [metabase.search.core :as search][metabase.task.core :as task][metabase.util.log :as log])(:import (java.time Duration Instant)@@ -29,20 +31,41 @@;; but if I do that quartz cannot initialize the job - there is probably a way around this.(defonce ^:private execution-thread-ref (volatile! nil))+(defn- hnsw-strategy? []+ (contains? search.config/hnsw-index-backed-strategies+ (semantic.settings/semantic-search-vector-strategy)))+(deftype ^{DisallowConcurrentExecution true:doc "Runs an indexer process for a time, expects to be rescheduled to continue"}SemanticSearchIndexer []org.quartz.Job(execute [_ _]- (when (semantic.u/semantic-search-available?)+ (when (semantic.u/semantic-search-active?)(log/with-context {:quartz-job-type 'SemanticSearchIndexer}- (when (search.engine/supported-engine? :search.engine/semantic)- (try- (vreset! execution-thread-ref (Thread/currentThread))- (semantic-search.indexer/quartz-job-run! (semantic.env/get-pgvector-datasource!) (semantic.env/get-index-metadata))- (finally- (locking execution-thread-ref- (vreset! execution-thread-ref nil))))))))+ (try+ (vreset! execution-thread-ref (Thread/currentThread))+ (let [pgvector (semantic.env/get-pgvector-datasource!)+ index-metadata (semantic.env/get-index-metadata)]+ (if-let [{:keys [index]} (semantic.index-metadata/get-active-index-state pgvector index-metadata)]+ (do+ ;; The strategy setter's build event no-ops while semantic is inactive, so an index-backed+ ;; strategy configured before (re)activation arrives here with no HNSW index. CONCURRENTLY+ ;; registers the index in pg_indexes as soon as the build starts, so this fires only once.+ (when (and (hnsw-strategy?)+ (not (semantic.u/index-exists? pgvector (semantic.index/hnsw-index-name index))))+ (semantic.core/build-hnsw-index-async!))+ (semantic-search.indexer/quartz-job-run! pgvector index-metadata))+ ;; Engines can activate at runtime (license applied, kill switch re-enabled,+ ;; additional-search-engines set on another node); initializing from the next tick heals+ ;; every such path within seconds. Initialize all active engines, not just semantic:+ ;; activation may also have activated dependencies with no index.+ (do+ (search/init-index!)+ (when (hnsw-strategy?)+ (semantic.core/build-hnsw-index-async!)))))+ (finally+ (locking execution-thread-ref+ (vreset! execution-thread-ref nil)))))))org.quartz.InterruptableJob(interrupt [_];; locking required here to avoid racing with the unset in the finally@@ -55,7 +78,7 @@(def ^:private ^Duration run-frequency (Duration/parse "PT20S"))(defmethod task/init! ::SemanticSearchIndexer [_]- (when (semantic.u/semantic-search-available?)+ (when (semantic.u/semantic-search-configured?)(let [job (jobs/build(jobs/of-type SemanticSearchIndexer)(jobs/store-durably)@@ -73,6 +96,5 @@;; Safety net: an instance that booted already configured for an HNSW-index-backed strategy (set via env;; var, or set before an active index existed) never saw the setter's transition event, so build the;; index now. Covers :hnsw and the :hnsw-iterative-* strategies, which all query through the index.- (when (contains? search.config/hnsw-index-backed-strategies- (semantic.settings/semantic-search-vector-strategy))+ (when (hnsw-strategy?)(semantic.core/build-hnsw-index-async!))))diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/task/metric_collector.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/task/metric_collector.cljindex da7e1006e45..b2e45e054c4 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/task/metric_collector.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/task/metric_collector.clj@@ -65,7 +65,9 @@(log/warn "DLQ table does not exist. Index may not have been initialized.")))(defn- collect-metrics! []- (when (semantic.u/semantic-search-available?)+ ;; Active, not merely available: on an available-but-inactive instance the index tables never exist,+ ;; and the collectors would warn about them on every run.+ (when (semantic.u/semantic-search-active?)(let [pgvector (semantic.env/get-pgvector-datasource!)index-metadata (semantic.env/get-index-metadata)](collect-gate-size! pgvector)@@ -80,7 +82,7 @@(defmethod task/init! ::SemanticMetricCollector[_]- (when (semantic.u/semantic-search-available?)+ (when (semantic.u/semantic-search-configured?)(let [job (jobs/build(jobs/of-type SemanticMetricCollector)(jobs/with-identity collector-job-key))diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/task/usage_trimmer.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/task/usage_trimmer.cljindex 2b04df57dae..9a8d4a7e9df 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/task/usage_trimmer.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/task/usage_trimmer.clj@@ -33,7 +33,7 @@(defmethod task/init! ::SemanticSearchUsageTrimmer[_]- (when (semantic.u/semantic-search-available?)+ (when (semantic.u/semantic-search-configured?)(let [job (jobs/build(jobs/of-type SemanticSearchUsageTrimmer)(jobs/with-identity trimmer-job-key))diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/util.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/util.cljindex 7bf0e5dba45..69fe3590a3b 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/util.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/util.clj@@ -1,7 +1,9 @@(ns metabase-enterprise.semantic-search.util(:require[metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource]+ [metabase-enterprise.semantic-search.settings :as semantic.settings][metabase.premium-features.core :as premium-features]+ [metabase.search.engine :as search.engine][next.jdbc :as jdbc][next.jdbc.result-set :as jdbc.rs]))@@ -23,8 +25,33 @@{:builder-fn jdbc.rs/as-unqualified-lower-maps})(:index_exists false)))+(defn semantic-search-configured?+ "Is a pgvector DB configured for this instance?+ The boot-static input: gates Quartz job scheduling, which happens once at startup.+ The runtime gates (license, kill switch, engine activity) are checked per job execution instead, so+ flipping them never requires a restart.+ This check becomes a DB probe under BOT-1796."+ []+ (string? (not-empty semantic.db.datasource/db-url)))++(defn semantic-search-capable?+ "Does this instance have the infrastructure for semantic search: the premium feature and a pgvector DB.+ Deliberately excludes the kill switch, which is checked per execution so it works at runtime."+ []+ ;; Cheap gate first: semantic-search-configured? becomes a DB probe under BOT-1796.+ (and (premium-features/has-feature? :semantic-search)+ (semantic-search-configured?)))+(defn semantic-search-available?- "Predicate to check whether semantic search is available on the instance."+ "Whether semantic search can run on this instance: capable and not disabled by the kill switch.+ Engine selection, hygiene tasks, and metrics key off this."+ []+ (and (semantic-search-capable?)+ (semantic.settings/semantic-search-enabled)))++(defn semantic-search-active?+ "Is the semantic index being maintained on this instance?+ Anything that writes to the index (or pays for embeddings) must gate on this, not on availability:+ a supported engine that is neither the default nor an additional engine has no index worth feeding."[]- (and (string? (not-empty semantic.db.datasource/db-url))- (premium-features/has-feature? :semantic-search)))+ (contains? (set (search.engine/active-engines)) :search.engine/semantic))diff --git a/enterprise/backend/test/metabase_enterprise/data_studio/search_test.clj b/enterprise/backend/test/metabase_enterprise/data_studio/search_test.cljindex d9e11ff700d..5407b91e2ba 100644--- a/enterprise/backend/test/metabase_enterprise/data_studio/search_test.clj+++ b/enterprise/backend/test/metabase_enterprise/data_studio/search_test.clj@@ -7,9 +7,17 @@[metabase.permissions.core :as perms][metabase.permissions.models.data-permissions :as data-perms][metabase.search.core :as search]+ [metabase.search.engine :as search.engine][metabase.test :as mt][metabase.util.random :refer [random-name]]))+(defn- test-engines+ "The engines to exercise: in-place always, appdb only where the app db supports it.+ An explicit search_engine the instance cannot serve returns a 400 rather than falling back."+ []+ (filter #(search.engine/supported-engine? (keyword "search.engine" %))+ ["in-place" "appdb"]))+(deftest published-table-collection-filter-test(mt/with-premium-features #{:library}(testing "Published tables are included in collection-filtered search"@@ -26,7 +34,7 @@:model/Table {table-4 :id} {:name "Root Published Table" :is_published true :collection_id nil}];; Initialize search index for appdb engine(search/reindex! {:async? false :in-place? true})- (doseq [engine ["in-place" "appdb"]]+ (doseq [engine (test-engines)](testing (str "with engine " engine)(testing "Global search"(let [results (mt/user-http-request :crowberto :get 200 "search" :search_engine engine)]@@ -85,7 +93,7 @@(mt/with-temp[:model/Table {root-table :id} {:name "Root Published Searchable" :is_published true :collection_id nil}](search/reindex! {:async? false :in-place? true})- (doseq [engine ["in-place" "appdb"]]+ (doseq [engine (test-engines)](testing (str "with engine " engine)(let [results (mt/user-http-request :crowberto :get 200 "search" :q "Root Published Searchable" :search_engine engine)](is (some #(= root-table (:id %)) (:data results))@@ -104,7 +112,7 @@:db_id db-id:is_published false}](search/reindex! {:async? false :in-place? true})- (doseq [engine ["in-place" "appdb"]]+ (doseq [engine (test-engines)](testing (str "with engine " engine)(let [results (mt/user-http-request :crowberto :get 200 "search":q "Context" :models "table" :search_engine engine)@@ -132,7 +140,7 @@;; Explicitly deny both view and query perms so this assertion doesn't depend on global defaults.(data-perms/set-database-permission! (perms/all-users-group) (mt/id) :perms/view-data :blocked)(data-perms/set-database-permission! (perms/all-users-group) (mt/id) :perms/create-queries :no)- (doseq [engine ["in-place" "appdb"]]+ (doseq [engine (test-engines)](search/reindex! {:async? false :in-place? true})(mt/with-non-admin-groups-no-root-collection-perms(let [results (mt/user-http-request :rasta :get 200 "search"@@ -157,7 +165,7 @@;; Grant data permissions on the table(data-perms/set-database-permission! (perms/all-users-group) (mt/id) :perms/view-data :unrestricted)(data-perms/set-database-permission! (perms/all-users-group) (mt/id) :perms/create-queries :query-builder)- (doseq [engine ["in-place" "appdb"]]+ (doseq [engine (test-engines)](search/reindex! {:async? false :in-place? true});; Remove collection permissions so the user has data perms but NO collection access(mt/with-non-admin-groups-no-root-collection-perms@@ -197,7 +205,7 @@:model/Card {outside-card :id} {:name (str search-term " Outside Card"):collection_id outside-coll}](search/reindex! {:async? false :in-place? true})- (doseq [engine ["in-place" "appdb"]]+ (doseq [engine (test-engines)](testing (str "with engine " engine)(let [results (mt/user-http-request :crowberto :get 200 "search":q search-term@@ -221,7 +229,7 @@;; Explicitly grant data/query perms so this does not rely on global test defaults.(data-perms/set-database-permission! (perms/all-users-group) (mt/id) :perms/view-data :unrestricted)(data-perms/set-database-permission! (perms/all-users-group) (mt/id) :perms/create-queries :query-builder)- (doseq [engine ["in-place" "appdb"]]+ (doseq [engine (test-engines)](search/reindex! {:async? false :in-place? true})(mt/with-non-admin-groups-no-root-collection-perms(let [result-ids (->> (mt/user-http-request :rasta :get 200 "search"@@ -257,7 +265,7 @@(perms/grant-collection-read-permissions! (perms/all-users-group) accessible)(perms/revoke-collection-permissions! (perms/all-users-group) no-access)(search/reindex! {:async? false :in-place? true})- (doseq [engine ["in-place" "appdb"]]+ (doseq [engine (test-engines)](testing (str "with engine " engine)(let [result-ids (->> (mt/user-http-request :rasta :get 200 "search":q search-term :search_engine engine)diff --git a/enterprise/backend/test/metabase_enterprise/metabot/tools/semantic_search_test.clj b/enterprise/backend/test/metabase_enterprise/metabot/tools/semantic_search_test.cljindex 572786b9adb..812f29e6ea0 100644--- a/enterprise/backend/test/metabase_enterprise/metabot/tools/semantic_search_test.clj+++ b/enterprise/backend/test/metabase_enterprise/metabot/tools/semantic_search_test.clj@@ -120,7 +120,7 @@(mt/with-test-user :rasta(semantic.tu/with-test-db! {:mode :mock-initialized}(with-and-without-semantic-search! test-mock-embeddings- (search.tu/with-new-search-and-legacy-search+ (search.tu/with-appdb-search-and-legacy-search(let [semantic-support? (search.engine/supported-engine? :search.engine/semantic)];; "belligerent" and "bellicose" are semantically similar to our search terms;; ("combative", "quarrelsome") but should NOT match since we're only doing keyword search@@ -146,7 +146,7 @@(mt/with-test-user :rasta(semantic.tu/with-test-db! {:mode :mock-initialized}(with-semantic-search-if-available! test-mock-embeddings- (search.tu/with-new-search-and-legacy-search+ (search.tu/with-appdb-search-and-legacy-search;; "belligerent" and "baseline" will match via keyword search (exact match in term-queries);; "ancillary" and "adjunct" will match via semantic search (similar embeddings);; "bellicose" and "quixotic" should NOT match (not in search terms)diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/api_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/api_test.cljindex 0863f72e177..133d1e3e221 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/api_test.clj@@ -56,29 +56,31 @@(deftest re-init-endpoint-test(testing "POST /api/search/re-init with semantic search support"(mt/with-premium-features #{:semantic-search}- (semantic.tu/with-test-db! {:mode :mock-indexed}- (let [original-index semantic.tu/mock-index- original-table-name (:table-name original-index)- new-index (mt/with-dynamic-fn-redefs [semantic.index/model-table-suffix (constantly 345)]- (#'semantic.pgvector-api/fresh-index semantic.tu/mock-index-metadata semantic.tu/mock-embedding-model :force-reset? true))- new-table-name (:table-name new-index)- pgvector (semantic.env/get-pgvector-datasource!)]- (is (semantic.tu/table-exists-in-db? original-table-name))- (is (not (semantic.tu/table-exists-in-db? new-table-name)))- (let [best-index (semantic.index-metadata/find-compatible-index! pgvector semantic.tu/mock-index-metadata semantic.tu/mock-embedding-model)]- (is (=? original-index (:index best-index)))- (is (:active best-index)))- (testing "re-init creates the new index"- (mt/with-dynamic-fn-redefs [semantic.index/model-table-suffix (constantly 345)]- (let [response (mt/user-http-request :crowberto :post 200 "search/re-init")]- (is (contains? response :message))))- (is (not= original-table-name new-table-name))+ ;; Semantic is not the default engine here, so it must be activated explicitly.+ (mt/with-temporary-setting-values [additional-search-engines ["semantic"]]+ (semantic.tu/with-test-db! {:mode :mock-indexed}+ (let [original-index semantic.tu/mock-index+ original-table-name (:table-name original-index)+ new-index (mt/with-dynamic-fn-redefs [semantic.index/model-table-suffix (constantly 345)]+ (#'semantic.pgvector-api/fresh-index semantic.tu/mock-index-metadata semantic.tu/mock-embedding-model :force-reset? true))+ new-table-name (:table-name new-index)+ pgvector (semantic.env/get-pgvector-datasource!)](is (semantic.tu/table-exists-in-db? original-table-name))- (is (semantic.tu/table-exists-in-db? new-table-name))- (is (zero? (semantic.tu/index-count new-index))))- (let [best-index (semantic.index-metadata/find-compatible-index! pgvector semantic.tu/mock-index-metadata semantic.tu/mock-embedding-model)]- (is (=? new-index (:index best-index)))- (is (:active best-index)))- (testing "Index can be populated after re-init"- (semantic.tu/upsert-index! (semantic.tu/mock-documents) :index new-index)- (is (pos? (semantic.tu/index-count new-index)))))))))+ (is (not (semantic.tu/table-exists-in-db? new-table-name)))+ (let [best-index (semantic.index-metadata/find-compatible-index! pgvector semantic.tu/mock-index-metadata semantic.tu/mock-embedding-model)]+ (is (=? original-index (:index best-index)))+ (is (:active best-index)))+ (testing "re-init creates the new index"+ (mt/with-dynamic-fn-redefs [semantic.index/model-table-suffix (constantly 345)]+ (let [response (mt/user-http-request :crowberto :post 200 "search/re-init")]+ (is (contains? response :message))))+ (is (not= original-table-name new-table-name))+ (is (semantic.tu/table-exists-in-db? original-table-name))+ (is (semantic.tu/table-exists-in-db? new-table-name))+ (is (zero? (semantic.tu/index-count new-index))))+ (let [best-index (semantic.index-metadata/find-compatible-index! pgvector semantic.tu/mock-index-metadata semantic.tu/mock-embedding-model)]+ (is (=? new-index (:index best-index)))+ (is (:active best-index)))+ (testing "Index can be populated after re-init"+ (semantic.tu/upsert-index! (semantic.tu/mock-documents) :index new-index)+ (is (pos? (semantic.tu/index-count new-index))))))))))diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/repair_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/repair_test.cljindex b3d5fff638d..4616911649a 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/repair_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/repair_test.clj@@ -5,7 +5,10 @@[honey.sql :as sql][java-time.api :as t][metabase-enterprise.semantic-search.core :as semantic.core]+ [metabase-enterprise.semantic-search.embedding :as semantic.embedding][metabase-enterprise.semantic-search.env :as semantic.env]+ [metabase-enterprise.semantic-search.index :as semantic.index]+ [metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata][metabase-enterprise.semantic-search.repair :as semantic.repair][metabase-enterprise.semantic-search.test-util :as semantic.tu][metabase-enterprise.semantic-search.util :as semantic.util]@@ -87,6 +90,25 @@(is (tombstone? deleted-card-entry) "Deleted card should be a tombstone in gate table")(is (tombstone? deleted-dashboard-entry) "Deleted dashboard should be a tombstone in gate table"))))))))))+(deftest repair-index-initializes-when-missing-test+ (testing "repair-index! initializes the index when none is active, so runtime activation backfills"+ (mt/with-premium-features #{:semantic-search}+ (semantic.tu/with-test-db! {:mode :blank}+ ;; with-redefs mirrors the :mock-initialized bindings, which include non-fn values that+ ;; with-dynamic-fn-redefs cannot proxy.+ #_{:clj-kondo/ignore [:metabase/prefer-with-dynamic-fn-redefs]}+ (with-redefs [semantic.embedding/get-configured-model (fn [] semantic.tu/mock-embedding-model)+ semantic.index-metadata/default-index-metadata semantic.tu/mock-index-metadata+ semantic.index/model-table-suffix semantic.tu/mock-table-suffix]+ (let [pgvector (semantic.env/get-pgvector-datasource!)+ index-metadata (semantic.env/get-index-metadata)]+ (is (nil? (semantic.index-metadata/get-active-index-state pgvector index-metadata)))+ (semantic.core/repair-index! [(create-test-document "card" 1 "Dog Training Guide")])+ (is (some? (semantic.index-metadata/get-active-index-state pgvector index-metadata)))+ (testing "the supplied documents are gated for backfill"+ (let [gate-contents (gate-table-contents pgvector (:gate-table-name index-metadata))]+ (is (some? (:document (gate-entry-by-id gate-contents "card_1"))))))))))))+(deftest repair-table-cleanup-test(testing "The repair table gets cleaned up properly at the end of a repair-index! job"(mt/with-premium-features #{:semantic-search}diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/task/indexer_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/task/indexer_test.cljindex 62b623b8d0c..b9aa772128a 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/task/indexer_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/task/indexer_test.clj@@ -16,7 +16,7 @@;; env var), where the setter's transition event never fired -- see the safety net in sut/init!. Redefs;; avoid scheduling a real quartz job or spawning a real build future.(let [builds (atom 0)]- (mt/with-dynamic-fn-redefs [semantic.u/semantic-search-available? (constantly true)+ (mt/with-dynamic-fn-redefs [semantic.u/semantic-search-configured? (constantly true)task/schedule-task! (fn [& _] nil)semantic.core/build-hnsw-index-async! (fn [] (swap! builds inc))](testing ":brute-force does not trigger a build"diff --git a/enterprise/backend/test/metabase_enterprise/tenants/search_test.clj b/enterprise/backend/test/metabase_enterprise/tenants/search_test.cljindex e82f4665832..5a2506b52a2 100644--- a/enterprise/backend/test/metabase_enterprise/tenants/search_test.clj+++ b/enterprise/backend/test/metabase_enterprise/tenants/search_test.clj@@ -14,7 +14,7 @@(use-fixtures :once (fixtures/initialize :db))(use-fixtures :each (fn [thunk] (binding [search.ingestion/*force-sync* true]- (search.tu/with-new-search-if-available-otherwise-legacy (thunk)))))+ (search.tu/with-appdb-search-if-available-otherwise-legacy (thunk)))))(deftest dedicated-tenant-collection-dataset-search-test(testing "Search returns datasets (models) from dedicated tenant collections"diff --git a/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/api_test.cljindex d9d7cfd7534..c0783a79d09 100644--- a/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj@@ -154,7 +154,7 @@:name query-name):model/Transform {python-id :id} (assoc (python-transform-map (str "target_" (u/generate-nano-id))):name python-name)]- (search.tu/with-new-search-and-legacy-search+ (search.tu/with-appdb-search-and-legacy-search(testing "no transforms feature"(mt/with-premium-features #{}(is (empty? (search-transform-ids search-term)))))@@ -175,7 +175,7 @@:name query-name):model/Transform {python-id :id} (assoc (python-transform-map (str "target_" (u/generate-nano-id))):name python-name)]- (search.tu/with-new-search-and-legacy-search+ (search.tu/with-appdb-search-and-legacy-search(mt/with-premium-features #{:transforms-basic :transforms-python :hosting}(is (= #{query-id python-id} (search-transform-ids search-term))))(mt/with-premium-features #{:transforms-basic :hosting}@@ -191,7 +191,7 @@query-name (str search-term "-query")](mt/with-temp [:model/Transform {query-id :id} (assoc (query-transform-payload (str "target_" (u/generate-nano-id))):name query-name)]- (search.tu/with-new-search-and-legacy-search+ (search.tu/with-appdb-search-and-legacy-search(let [ids (search-api-transform-ids :crowberto search-term)](is (empty? ids))(is (not (contains? ids query-id)))))))))))diff --git a/src/metabase/metabot/self/features.clj b/src/metabase/metabot/self/features.cljindex b5a48a21ceb..ee5a22b5178 100644--- a/src/metabase/metabot/self/features.clj+++ b/src/metabase/metabot/self/features.clj@@ -9,7 +9,9 @@(def feature-predicates"Registry of feature keywords to predicate functions."- {:semantic-search #(search.engine/supported-engine? :search.engine/semantic)})+ ;; Status, not capability: advertising semantic_queries in tool schemas when no semantic index is+ ;; maintained makes the LLM emit semantic queries that would run as keyword searches.+ {:semantic-search #(= :ok (search.engine/engine-status :search.engine/semantic))})(defn feature-available?"Check if a feature is available. Unknown features return true (fail-open)."diff --git a/src/metabase/metabot/tools/search.clj b/src/metabase/metabot/tools/search.cljindex 6f833587179..f73b113e58b 100644--- a/src/metabase/metabot/tools/search.clj+++ b/src/metabase/metabot/tools/search.clj@@ -307,7 +307,7 @@semantic? #{:search.engine/semantic}semantic-engine (u/seek semantic? (search.engine/active-engines))fallback-engine (when semantic-engine- (u/seek (comp not semantic?) (search.engine/supported-engines)))+ (search.engine/fallback-engine semantic-engine))fused-results (if semantic-engine;; Perform semantic and non-semantic search respectively, then fuse results.(reciprocal-rank-fusiondiff --git a/src/metabase/search/api.clj b/src/metabase/search/api.cljindex cae6faa6021..f7d21b50c92 100644--- a/src/metabase/search/api.clj+++ b/src/metabase/search/api.clj@@ -11,6 +11,7 @@[metabase.request.core :as request][metabase.search.config :as search.config][metabase.search.core :as search]+ [metabase.search.engine :as search.engine][metabase.search.ingestion :as ingestion][metabase.search.settings :as search.settings][metabase.search.task.search-index :as task.search-index]@@ -26,7 +27,8 @@(set! *warn-on-reflection* true)-(def ^:private engine-cookie-name "metabase.SEARCH_ENGINE")+;; Renamed from metabase.SEARCH_ENGINE so long-lived cookies written before engine validation existed stay inert.+(def ^:private engine-cookie-name "metabase.SEARCH_ENGINE_OVERRIDE")(defn- cookie-expiry [];; 20 years should be long enough to trial an experimental search engine@@ -42,6 +44,16 @@:path "/":expires (cookie-expiry)}))))+(defn- clear-engine-cookie! [respond]+ (fn [response]+ (respond+ (response/set-cookie response+ engine-cookie-name+ ""+ {:http-only true+ :path "/"+ :max-age 0}))))+(defn- process-non-temporal-dim-ids"Parse and process non-temporal dimension IDs JSON string.Filters out null values and sorts ascending, returning as JSON string."@@ -57,15 +69,65 @@(log/warn "Failed to parse non-temporal dimension IDs:" (ex-message e))nil))))+(defn- param->engine+ "Parse a search_engine param or cookie value into an engine keyword, nil when blank.+ Malformed non-blank values parse to an unknown engine rather than nil, so explicit requests 400."+ [value]+ (when-not (str/blank? value)+ (search.engine/canonical-engine value)))++(defn- check-engine-serves!+ "400 when an explicitly requested engine cannot serve searches, naming the cause."+ [engine]+ (case (search.engine/engine-status engine)+ :unknown+ (throw (ex-info (tru "Unknown search engine: {0}" (name engine)) {:status-code 400}))+ :unsupported+ (throw (ex-info (tru "Search engine {0} is not supported on this instance" (name engine)) {:status-code 400}))+ :inactive+ (throw (ex-info (tru "Search engine {0} is not enabled; add it to additional-search-engines to use it"+ (name engine))+ {:status-code 400}))+ :ok nil))++(defn- cookie-engine+ "The engine cookie's engine name when it can still serve, else nil.+ Cookies outlive configuration, so a stale value degrades to the default instead of erroring."+ [request]+ (when-let [engine (param->engine (get-in request [:cookies engine-cookie-name :value]))]+ (when (= :ok (search.engine/engine-status engine))+ (name engine))))+(defn- +engine-cookie [handler](open-api/handler-with-open-api-spec(fn [request respond raise]- (if-let [new-engine (get-in request [:params :search_engine])]- (handler request (set-engine-cookie! respond new-engine) raise)- (handler (->> (get-in request [:cookies engine-cookie-name :value])- (assoc-in request [:params :search_engine]))- respond- raise)))+ ;; Endpoints read :query-params (string keys), so that is where the engine must be injected.+ (let [raw (get-in request [:query-params "search_engine"])]+ (cond+ ;; A repeated query param parses as a vector: pass it through for schema validation to reject.+ (and raw (not (string? raw)))+ (handler request respond raise)++ ;; An explicit blank unpins: delete the engine cookie and resolve the default.+ (and raw (str/blank? raw))+ (handler (assoc-in request [:query-params "search_engine"] nil)+ (clear-engine-cookie! respond)+ raise)++ :else+ (if-let [engine (param->engine raw)]+ (try+ (check-engine-serves! engine)+ (handler (assoc-in request [:query-params "search_engine"] (name engine))+ (set-engine-cookie! respond (name engine))+ raise)+ (catch Exception e+ (raise e)))+ (let [cookie (cookie-engine request)]+ (handler (cond-> request+ cookie (assoc-in [:query-params "search_engine"] cookie))+ respond+ raise))))))(fn [prefix](open-api/open-api-spec handler prefix))))diff --git a/src/metabase/search/appdb/core.clj b/src/metabase/search/appdb/core.cljindex fd02e9c96f3..1669c7eb7ae 100644--- a/src/metabase/search/appdb/core.clj+++ b/src/metabase/search/appdb/core.clj@@ -16,7 +16,6 @@[metabase.search.impl :as search.impl][metabase.search.ingestion :as search.ingestion][metabase.search.permissions :as search.permissions]- [metabase.search.settings :as search.settings][metabase.search.spec :as search.spec][metabase.search.util :as search.util][metabase.settings.core :as setting]@@ -45,13 +44,7 @@#{:postgres :h2})(defmethod search.engine/supported-engine? :search.engine/appdb [_]- (and (or config/is-dev?- ;; TODO (Chris 2025-11-07) This backwards dependency is unfortunate, we should find a better solution.- ;; Perhaps just an explicit setting for enabling it.- ;; This also opens us up to swapping out the fallback, e.g. to elastic search.- ;; if the default engine is semantic we want appdb to be available, as we want to mix results- (#{"appdb" "semantic"} (some-> (search.settings/search-engine) name)))- (supported-db? (mdb/db-type))))+ (supported-db? (mdb/db-type)))(defmethod search.engine/disjunction :search.engine/appdb [_ terms](when (seq terms)diff --git a/src/metabase/search/core.clj b/src/metabase/search/core.cljindex d173c740573..879be246b09 100644--- a/src/metabase/search/core.clj+++ b/src/metabase/search/core.clj@@ -84,7 +84,10 @@(defmethod analytics.core/initial-value :metabase-search/engine-active[_ {:keys [engine]}]- (if (search.engine/supported-engine? (keyword "search.engine" engine)) 1 0))+ ;; Can the engine serve queries: in-place always can, indexed engines only while their index is maintained.+ (if (= :ok (search.engine/engine-status (keyword "search.engine" engine)))+ 1+ 0))(defn supports-index?"Does this instance support a search index, of any sort?"@@ -94,7 +97,8 @@(defn init-index!"Ensure there is an index ready to be populated."[& {:as opts}]- (when (supports-index?)+ (search.engine/log-resolution!)+ (when-let [engines (seq (search.engine/active-engines))](log/info "Initializing search indexes")(tracing/with-span :search "search.init-index" {}(lib-be/with-metadata-provider-cache@@ -103,7 +107,7 @@(let [timer (u/start-timer)report (reduce (partial merge-with max)nil- (for [e (search.engine/active-engines)]+ (for [e engines](search.engine/init! e opts)))duration (u/since-ms timer)](if (seq report)@@ -120,7 +124,7 @@(throw e)))))))(defn- reindex-logic! [opts]- (when (supports-index?)+ (when-let [engines (seq (search.engine/active-engines))](tracing/with-span :search "search.reindex" {}(lib-be/with-metadata-provider-cache(try@@ -128,7 +132,7 @@(let [timer (u/start-timer)report (reduce (partial merge-with max)nil- (for [e (search.engine/active-engines)]+ (for [e engines](search.engine/reindex! e opts)))duration (u/since-ms timer)](analytics/inc! :metabase-search/index-reindex-ms duration)diff --git a/src/metabase/search/engine.clj b/src/metabase/search/engine.cljindex 001f5d6e8dc..068dc5d600b 100644--- a/src/metabase/search/engine.clj+++ b/src/metabase/search/engine.clj@@ -1,16 +1,22 @@;; The interface encapsulating the various search engine backends.(ns metabase.search.engine(:require- [metabase.search.settings :as settings]))+ [clojure.string :as str]+ [metabase.search.settings :as settings]+ [metabase.util :as u]+ [metabase.util.log :as log]))(def ^:private default-engine-precedence- "In the absence of explicit configuration, these are the engines to try using, in decreasing order of preference."+ "The engines to use as the default, in decreasing order of preference.+ The first supported one wins, unless overridden by the [[settings/search-engine]] setting."[:search.engine/semantic:search.engine/appdb:search.engine/in-place])(defmulti supported-engine?- "Does this instance support the given engine?"+ "Can this instance run the given engine?+ Implementations must be pure capability checks: app-db type, premium features, required infrastructure.+ They must not read [[settings/search-engine]], which selects among the supported engines."{:arglists '([engine])}identity)@@ -18,6 +24,14 @@(throw (ex-info (format "Unknown search engine: %s" engine){:engine engine})))+(defmulti dependencies+ "Engines whose indexes this engine needs in order to serve queries.+ Semantic, for example, mixes appdb results into its own and falls back to appdb when its index is unavailable."+ {:arglists '([engine])}+ identity)++(defmethod dependencies :default [_] nil)+(defmulti results"Return a reducible of the search result matching a given query."{:arglists '([search-context])}@@ -97,25 +111,71 @@;; If we end up with more "abstract" nodes, we may want a better way to filter them out.(keys (dissoc (methods supported-engine?) :default)))+(defn known-engine?+ "Is the engine, or an engine it derives from, registered?"+ [engine]+ (let [registered? #(contains? (methods supported-engine?) %)]+ (boolean (some registered? (cons engine (ancestors engine))))))++(def ^:private warned-engine-values+ "Values already warned about, so misconfiguration warns once rather than on every call."+ (atom #{}))++(defn- warn-once! [value message]+ (when-not (contains? @warned-engine-values value)+ (swap! warned-engine-values conj value)+ (log/warn message)))++(def ^:private legacy-engine-names+ "Renamed engines that may still arrive via old configuration or long-lived cookies."+ {"fulltext" "appdb"})++(defn canonical-engine+ "Coerce an engine name (string or keyword) to its canonical engine keyword.+ Resolves legacy names and tolerates the qualified search.engine/ prefix the API echoes in responses.+ Every configuration and request boundary must canonicalize through this, so that engine identities+ compare with =.+ Malformed values yield an unknown engine named after the raw value, never an empty keyword."+ [value]+ (let [raw (name value)+ stripped (str/replace raw #"^search\.engine/" "")+ engine-name (if (str/blank? stripped) raw stripped)]+ (keyword "search.engine" (get legacy-engine-names engine-name engine-name))))++(defn- validated-engine+ "Coerce a configured engine name to a known engine keyword.+ Unknown values return nil with a one-time warning, so a typo in MB_SEARCH_ENGINE cannot break search."+ [value]+ (when value+ (let [engine (canonical-engine value)]+ (if (known-engine? engine)+ engine+ (warn-once! value (format "Ignoring unknown search engine: %s" value))))))++(defn- configured-engine []+ ;; The raw setting value, not the public getter: [[settings/search-engine]] resolves its default via+ ;; [[default-engine]], so reading it here would recurse.+ (validated-engine (settings/configured-search-engine)))+(defn supported-engines"List the search engines that are supported, in order of usage preference.The configured engine comes first, if it is supported."[]- (let [configured-engine (some->> (settings/search-engine) name (keyword "search.engine"))- potential-engines (cond->> default-engine-precedence configured-engine (cons configured-engine))]- (distinct (filter supported-engine? potential-engines))))--(defn active-engines- "A list of supported search engines for which we will maintain an index, in order of usage preference.- Excludes :search.engine/in-place, which does not use an index."+ (let [configured (configured-engine)+ potential-engines (cond->> default-engine-precedence+ configured (cons configured))]+ ;; Drop engines that are not yet registered: the search-engine setting can be read before+ ;; metabase.search.init loads the engine implementations, and supported-engine? throws on unknown engines.+ (distinct (filter supported-engine? (filter known-engine? potential-engines)))))++(defn- additional-engines+ "The supported engines force-enabled by [[settings/additional-search-engines]]."[]- (remove #{:search.engine/in-place} (supported-engines)))--(defn known-engine?- "Is the given engine recognized?"- [engine]- (let [registered? #(contains? (methods supported-engine?) %)]- (some registered? (cons engine (ancestors engine)))))+ (->> (settings/additional-search-engines)+ (map str/trim)+ (remove str/blank?)+ (keep validated-engine)+ (filter supported-engine?)))(defmulti disjunction"Given multiple terms to search for, reduce this to a search expression that matches any of them in a single search.@@ -124,10 +184,51 @@(fn [search-engine _terms] search-engine))(defn default-engine- "In the absence of an explicit engine argument in a request, which engine should be used?"+ "The engine that serves requests with no explicit engine argument.+ The configured override when supported, otherwise the first supported engine in the precedence."+ []+ (let [configured (configured-engine)+ default (first (supported-engines))]+ (when (and configured (not= configured default))+ (warn-once! [:unsupported configured]+ (format "Configured search engine %s is not supported on this instance, using %s instead"+ configured default)))+ default))++(defn log-resolution!+ "Log the engine that serves search."[]- ;; TODO (Chris 2025-11-07) It would be good to have a warning on start up whenever this is *not* what's configured.- (first (supported-engines)))+ (log/infof "Search will be served by the %s engine" (default-engine)))++(defn active-engines+ "The engines for which we maintain an index, default engine first.+ The default engine and any [[settings/additional-search-engines]], plus their supported [[dependencies]].+ Excludes :search.engine/in-place, which does not use an index."+ []+ (if-let [default (default-engine)]+ (->> (concat [default] (additional-engines))+ (mapcat #(cons % (filter supported-engine? (filter known-engine? (dependencies %)))))+ distinct+ (remove #{:search.engine/in-place}))+ []))++(defn engine-status+ "Whether the engine can serve queries, and if not, why: :ok, :unknown, :unsupported, or :inactive.+ In-place needs no index, so it serves whenever supported; other engines must be in [[active-engines]].+ Membership is exact: callers must resolve engine aliases before asking."+ [engine]+ (cond+ (not (known-engine? engine)) :unknown+ (not (supported-engine? engine)) :unsupported+ (or (= engine :search.engine/in-place)+ (contains? (set (active-engines)) engine)) :ok+ :else :inactive))++(defn fallback-engine+ "The engine to mix results with and fall back to when `engine` cannot serve alone.+ Prefers engines with a maintained index over merely supported ones."+ [engine]+ (u/seek #(not= engine %) (concat (active-engines) (supported-engines))))(defmethod disjunction :default [_ terms] terms)diff --git a/src/metabase/search/semantic/core.clj b/src/metabase/search/semantic/core.cljindex e3a4dd2c5c0..c4225889d69 100644--- a/src/metabase/search/semantic/core.clj+++ b/src/metabase/search/semantic/core.clj@@ -67,6 +67,9 @@(log/warn e "Semantic search engine not supported")false)))+(defmethod search.engine/dependencies :search.engine/semantic [_]+ [:search.engine/appdb])+(defmethod search.engine/results :search.engine/semantic[search-ctx](results search-ctx))diff --git a/src/metabase/search/settings.clj b/src/metabase/search/settings.cljindex 5af7155ad69..48051731a9c 100644--- a/src/metabase/search/settings.clj+++ b/src/metabase/search/settings.clj@@ -1,8 +1,10 @@(ns metabase.search.settings(:require+ [metabase.app-db.core :as mdb][metabase.appearance.core :as appearance][metabase.settings.core :as setting :refer [defsetting]]- [metabase.util.i18n :as i18n]))+ [metabase.util.i18n :as i18n]+ [metabase.util.log :as log]))(defsetting search-typeahead-enabled(i18n/deferred-tru "Enable typeahead search in the {0} navbar?"@@ -13,15 +15,44 @@:export? true:audit :getter)+(defn configured-search-engine+ "The operator's engine override, or nil when unset.+ [[search-engine]] resolves a default when unset; use this to see what was explicitly configured."+ []+ (setting/get-value-of-type :keyword :search-engine))+(defsetting search-engine- (i18n/deferred-tru "Which engine to use by default for search. Supported values are :in-place, :appdb, and :semantic")+ (i18n/deferred-tru "The engine that serves search. Supported values are :in-place, :appdb, and :semantic. When unset, the best supported engine is used."):visibility :authenticated:export? false:setter :none- ;; TODO (Chris 2025-11-07) Would be good to remove the default and just use [search.engine/default-engine-precedence]- :default :appdb+ ;; Reading resolves the engine that actually serves search; [[configured-search-engine]] is the raw override.+ :getter (fn []+ (some-> ((requiring-resolve 'metabase.search.engine/default-engine)) name keyword)):type :keyword)+(defsetting additional-search-engines+ (i18n/deferred-tru "Engines to keep active (indexed and queryable per-request) in addition to the default engine.")+ :visibility :internal+ :export? false+ :encryption :no+ :default nil+ :type :csv+ ;; Post-commit + future: the trigger check must observe the final committed settings state (a surrounding+ ;; set-many! may include the setting that makes the engine supported) and stay off the transaction's thread.+ :setter (fn [new-value]+ (let [before (set ((requiring-resolve 'metabase.search.engine/active-engines)))+ result (setting/set-value-of-type! :csv :additional-search-engines new-value)]+ (mdb/do-after-commit+ #(future+ (try+ ((requiring-resolve 'metabase.search.task.search-index/trigger-init-for-newly-active-engines!)+ before)+ (catch Throwable t+ (log/error t "Failed to trigger search index init for newly active engines")))))+ result))+ :doc false)+(defsetting experimental-search-weight-overrides(i18n/deferred-tru "Used to override weights used for search ranking"):visibility :internaldiff --git a/src/metabase/search/task/search_index.clj b/src/metabase/search/task/search_index.cljindex bedbfda5697..696d5559601 100644--- a/src/metabase/search/task/search_index.clj+++ b/src/metabase/search/task/search_index.clj@@ -5,6 +5,7 @@[clojurewerkz.quartzite.triggers :as triggers][metabase.app-db.cluster-lock :as cluster-lock][metabase.search.core :as search]+ [metabase.search.engine :as search.engine][metabase.search.ingestion :as ingestion][metabase.startup.core :as startup][metabase.task.core :as task]@@ -45,6 +46,11 @@(cluster-lock/with-cluster-lock cluster-lock-name(search/reindex! {:async? false})))+(task/defjob ^{DisallowConcurrentExecution true+ :doc "Ensure indexes exist for the active engines"}+ SearchIndexInit [_ctx]+ (init!))+;; Atom holding a promise that is delivered when the background init thread finishes.;; nil when no init has been started — [[wait-for-init!]] returns immediately in that case.(defonce ^:private init-promise (atom nil))@@ -65,6 +71,24 @@(deliver p true))))).start)))+(defn trigger-init-for-newly-active-engines!+ "Trigger the init job if any engine beyond `before` is now active.+ A newly activated engine needs its index initialized before it can serve; already-active engines are+ skipped because semantic init re-gates every searchable document.+ No-op when the init job is not registered, i.e. the scheduler is not running (tests)."+ [before]+ (when (and (seq (remove before (search.engine/active-engines)))+ (task/job-exists? init-job-key))+ (task/trigger-now! init-job-key)))++;; A durable job with no schedule: triggered on demand when an engine is activated at runtime,+;; e.g. by the additional-search-engines setter.+(defmethod task/init! ::SearchIndexInit [_]+ (task/add-job! (jobs/build+ (jobs/of-type SearchIndexInit)+ (jobs/store-durably)+ (jobs/with-identity init-job-key))))+(defmethod task/init! ::SearchIndexReindex [_](let [job (jobs/build(jobs/of-type SearchIndexReindex)diff --git a/test/metabase/agent_api/api_test.clj b/test/metabase/agent_api/api_test.cljindex c7188ee92da..2d6aeebfa6d 100644--- a/test/metabase/agent_api/api_test.clj+++ b/test/metabase/agent_api/api_test.clj@@ -97,7 +97,7 @@(deftest search-test(binding [search.ingestion/*force-sync* true]- (search.tu/with-new-search-if-available-otherwise-legacy+ (search.tu/with-appdb-search-if-available-otherwise-legacy(mt/with-temp [:model/Table _ {:name "AgentSearchTestTable"}](testing "Returns search results for term queries"(is (=? {:data [{:type "table" :name "AgentSearchTestTable"}]@@ -108,7 +108,7 @@(deftest search-content-types-test(testing "search surfaces saved questions, dashboards, and collections (not just tables/metrics/models)"(binding [search.ingestion/*force-sync* true]- (search.tu/with-new-search-if-available-otherwise-legacy+ (search.tu/with-appdb-search-if-available-otherwise-legacy(mt/with-temp [:model/Card _ {:name "AgentSearchAcmeQuestion"}:model/Dashboard _ {:name "AgentSearchAcmeDashboard"}:model/Collection _ {:name "AgentSearchAcmeCollection"}]@@ -556,7 +556,7 @@(deftest search-finds-metrics-test(binding [search.ingestion/*force-sync* true]- (search.tu/with-new-search-if-available-otherwise-legacy+ (search.tu/with-appdb-search-if-available-otherwise-legacy(mt/with-temp [:model/Card _metric {:name "AgentSearchTestMetric":type :metric:database_id (mt/id)@@ -569,7 +569,7 @@(deftest search-finds-models-test(binding [search.ingestion/*force-sync* true]- (search.tu/with-new-search-if-available-otherwise-legacy+ (search.tu/with-appdb-search-if-available-otherwise-legacy(mt/with-temp [:model/Card _model {:name "AgentSearchTestModel":type :model:database_id (mt/id)diff --git a/test/metabase/analytics/prometheus_test.clj b/test/metabase/analytics/prometheus_test.cljindex ac7118ab845..3b689466183 100644--- a/test/metabase/analytics/prometheus_test.clj+++ b/test/metabase/analytics/prometheus_test.clj@@ -6,7 +6,7 @@[clojure.test :refer :all][iapetos.registry :as registry][metabase.analytics.prometheus :as prometheus]- [metabase.search.core :as search]+ [metabase.search.engine :as search.engine][metabase.test :as mt][metabase.test.fixtures :as fixtures][metabase.util :as u])@@ -219,16 +219,17 @@sum (fn [metric] (reduce + 0 (vals (engine->value metric))))](testing "A consistent set of engines is enumerated"(is (= (engines :metabase-search/engine-active)- (engines :metabase-search/engine-active))))+ (engines :metabase-search/engine-default))))(testing "The values are boolean"(is (set/superset? #{0 1} (set (vals (engine->value :metabase-search/engine-active)))))(is (set/superset? #{0 1} (set (vals (engine->value :metabase-search/engine-default))))))- (testing "Legacy search is always active"+ (testing "The default engine is active"+ (is (= 1 (value :metabase-search/engine-active (search.engine/default-engine)))))+ (testing "In-place can always serve, so it is always active"(is (= 1 (value :metabase-search/engine-active :in-place))))- (testing "There is at least one other active engine iff we support an index."- (if (search/supports-index?)- (is (< 1 (sum :metabase-search/engine-active)))- (is (= 1 (sum :metabase-search/engine-active)))))+ (testing "Beyond in-place, engines are active iff their index is maintained"+ (is (= (inc (count (remove #{:search.engine/in-place} (search.engine/active-engines))))+ (sum :metabase-search/engine-active))))(testing "There is only one default"(is (= 1 (sum :metabase-search/engine-default))))))diff --git a/test/metabase/documents/search_test.clj b/test/metabase/documents/search_test.cljindex 0b2a92f1335..cfabdceadd4 100644--- a/test/metabase/documents/search_test.clj+++ b/test/metabase/documents/search_test.clj@@ -133,7 +133,7 @@;; the legacy in-place engine LIKE-matches the raw prose-mirror JSON.(mt/with-temp [:model/Document {doc-id :id} {:name "Annual Summary":document (documents.test-util/text->prose-mirror-ast "quarterly revenue projections and growth")}]- (search.tu/with-new-search-and-legacy-search+ (search.tu/with-appdb-search-and-legacy-search(testing "found by a term that appears only in the body"(let [results (mt/user-http-request :crowberto :get 200 "search" :q "projections" :models "document")](is (contains? (set (map :id (:data results))) doc-id))))@@ -153,7 +153,7 @@:content [{:type "text" :text "see"}{:type "smartLink":attrs {:model "card" :entityId "abc" :label "Customer Retention"}}]}]}}]- (search.tu/with-new-search-and-legacy-search+ (search.tu/with-appdb-search-and-legacy-search(testing "found by a term that appears only in the smart-link label"(let [results (mt/user-http-request :crowberto :get 200 "search" :q "Retention" :models "document")](is (contains? (set (map :id (:data results))) doc-id))))))))diff --git a/test/metabase/metabot/curation_test.clj b/test/metabase/metabot/curation_test.cljindex aa3e8649fee..daa9a1b93d6 100644--- a/test/metabase/metabot/curation_test.clj+++ b/test/metabase/metabot/curation_test.clj@@ -11,7 +11,7 @@(deftest curated-ids-consistent-with-index-test(testing "curated-ids reads the same verdict from source-of-truth tables that the search indexprecomputes from the specs, so the two definitions stay consistent (BOT-1570)"- (search.tu/with-new-search-if-available-without-fallback+ (search.tu/with-appdb-search-if-available-without-fallback(mt/with-temp [:model/Database {db :id} {}:model/Collection {off :id} {:authority_level "official"};; name avoids the "curationfixture" search token so the collection itself isn't adiff --git a/test/metabase/search/api_test.clj b/test/metabase/search/api_test.cljindex 1c07d28b79e..ca0d2fc64df 100644--- a/test/metabase/search/api_test.clj+++ b/test/metabase/search/api_test.clj@@ -17,10 +17,12 @@[metabase.permissions.core :as perms][metabase.permissions.util :as perms-util][metabase.revisions.models.revision :as revision]+ [metabase.search.api :as search.api][metabase.search.appdb.core :as search.engines.appdb][metabase.search.appdb.index :as search.index][metabase.search.config :as search.config][metabase.search.core :as search]+ [metabase.search.engine :as search.engine][metabase.search.ingestion :as search.ingestion][metabase.search.test-util :as search.tu][metabase.test :as mt]@@ -40,7 +42,7 @@(def ^:private default-collection {:id false :name nil :authority_level nil :type nil})(use-fixtures :each (fn [thunk] (binding [search.ingestion/*force-sync* true]- (search.tu/with-new-search-if-available-otherwise-legacy (thunk)))))+ (search.tu/with-appdb-search-if-available-otherwise-legacy (thunk)))))(def ^:private default-search-row{:archived false@@ -334,6 +336,85 @@(is (=? {:engine string?}(mt/user-http-request :crowberto :get 200 "search" :q "x" :context "search-app"))))))+(deftest explicit-engine-validation-test+ (testing "an explicit search_engine that cannot serve returns a 400 naming the reason"+ (testing "unknown engine"+ (is (= "Unknown search engine: elastic"+ (mt/user-http-request :crowberto :get 400 "search" :q "x" :search_engine "elastic"))))+ (testing "known engine that this instance does not support"+ (is (= "Search engine semantic is not supported on this instance"+ (mt/user-http-request :crowberto :get 400 "search" :q "x" :search_engine "semantic"))))+ (testing "a malformed engine value is rejected, not treated as absent"+ (is (= "Unknown search engine: search.engine/"+ (mt/user-http-request :crowberto :get 400 "search" :q "x" :search_engine "search.engine/"))))+ (when (search.engine/supported-engine? :search.engine/appdb)+ (testing "supported engine that is neither the default nor an additional engine"+ ;; Checked below the HTTP layer: this namespace's :each fixture pins the default engine to appdb,+ ;; so no engine can be made inactive in a real request here.+ (mt/with-dynamic-fn-redefs [search.engine/active-engines (constantly [])]+ (is (thrown-with-msg? Exception+ #"not enabled"+ (#'search.api/check-engine-serves! :search.engine/appdb)))))))+ (testing "a servable explicit engine is accepted"+ (is (=? {:engine "search.engine/in-place"}+ (mt/user-http-request :crowberto :get 200 "search" :q "x" :search_engine "in-place"))))+ (testing "a blank engine is treated as absent"+ (is (=? {:engine string?}+ (mt/user-http-request :crowberto :get 200 "search" :q "x" :search_engine ""))))+ (testing "the fully qualified form the API returns in :engine is accepted"+ (is (=? {:engine "search.engine/in-place"}+ (mt/user-http-request :crowberto :get 200 "search" :q "x" :search_engine "search.engine/in-place"))))+ (testing "a repeated search_engine param fails validation rather than erroring"+ (is (=? {:errors {:search_engine some?}}+ (mt/user-http-request :crowberto :get 400 "search" :q "x" :search_engine ["appdb" "in-place"])))))++(def ^:private engine-cookie-name @#'search.api/engine-cookie-name)++(deftest engine-cookie-staleness-test+ (testing "a cookie engine that can still serve is honored"+ ;; in-place serves on every app db+ (is (= "in-place" (#'search.api/cookie-engine {:cookies {engine-cookie-name {:value "in-place"}}}))))+ (testing "a cookie engine that cannot serve degrades to the default"+ ;; appdb is either unsupported on this app db, or supported with no maintained index+ (mt/with-dynamic-fn-redefs [search.engine/active-engines (constantly [])]+ (is (nil? (#'search.api/cookie-engine {:cookies {engine-cookie-name {:value "appdb"}}})))))+ (testing "an unknown cookie engine degrades to the default"+ (is (nil? (#'search.api/cookie-engine {:cookies {engine-cookie-name {:value "wut"}}})))))++(deftest engine-cookie-request-test+ (let [pinned {:request-options {:cookies {engine-cookie-name {:value "in-place"}}}}]+ (testing "a pinned engine cookie selects the engine"+ (is (=? {:engine "search.engine/in-place"}+ (mt/user-http-request :crowberto :get 200 "search" pinned :q "x"))))+ (testing "an explicit blank search_engine unpins: the cookie is ignored and the default serves"+ (is (=? {:engine (u/qualified-name (search.engine/default-engine))}+ (mt/user-http-request :crowberto :get 200 "search" pinned :q "x" :search_engine ""))))))++(deftest engine-cookie-round-trip-test+ ;; The in-process client is required here: it runs the handler on the test thread, inside the fixture's+ ;; engine and index bindings.+ (let [cookie-pattern (re-pattern (str (java.util.regex.Pattern/quote engine-cookie-name) "=([^;]*)"))+ engine-cookie (fn [response]+ (some #(second (re-find cookie-pattern %))+ (u/one-or-many (get-in response [:headers "Set-Cookie"]))))]+ (testing "an explicit engine pins: the response sets the engine cookie"+ (let [response (mt/user-http-request-full-response :crowberto :get 200 "search" :q "x" :search_engine "in-place")]+ (is (= "in-place" (engine-cookie response)))+ (is (=? {:engine "search.engine/in-place"} (:body response)))))+ (testing "a request carrying the cookie serves the pinned engine"+ (is (=? {:engine "search.engine/in-place"}+ (mt/user-http-request :crowberto :get 200 "search"+ {:request-options {:cookies {engine-cookie-name {:value "in-place"}}}}+ :q "x"))))+ (testing "a blank explicit engine unpins: the response deletes the cookie and the default serves"+ (let [response (mt/user-http-request-full-response :crowberto :get 200 "search"+ {:request-options {:cookies {engine-cookie-name {:value "in-place"}}}}+ :q "x" :search_engine "")]+ (is (= "" (engine-cookie response)))+ (is (str/includes? (str/join " " (u/one-or-many (get-in response [:headers "Set-Cookie"])))+ "Max-Age=0"))+ (is (=? {:engine (u/qualified-name (search.engine/default-engine))} (:body response)))))))+(deftest vector-search-knobs-test(testing "the vector-search tuning/diagnostic knobs are admin-only"(doseq [[param value] {:vector_search_ef_search 100@@ -436,18 +517,14 @@(let [resp (search-request :crowberto :q "test" :search_engine "appdb" :limit 1)];; The index is not populated here, so there's not much interesting to assert.(is (= "search.engine/appdb" (:engine resp))))))- (testing "It can use the old search engine name, e.g. for old cookies"+ (testing "It normalizes the old search engine name, e.g. for old cookies"(search/init-index! {:force-reset? false :re-populate? false})(with-search-items-in-root-collection "test"(let [resp (search-request :crowberto :q "test" :search_engine "fulltext" :limit 1)]- (is (= "search.engine/fulltext" (:engine resp))))))- (testing "It will not use an unknown search engine"- (search/init-index! {:force-reset? false :re-populate? false})- (with-search-items-in-root-collection "test"- (let [resp (search-request :crowberto :q "test" :search_engine "wut" :limit 1)]- (is (#{"search.engine/in-place"- "search.engine/appdb"}- (:engine resp))))))))+ (is (= "search.engine/appdb" (:engine resp))))))+ (testing "It rejects an unknown search engine"+ (is (= "Unknown search engine: wut"+ (mt/user-http-request :crowberto :get 400 "search" :q "test" :search_engine "wut" :limit 1))))))(defn- get-available-models [& args](disjdiff --git a/test/metabase/search/engine_test.clj b/test/metabase/search/engine_test.cljnew file mode 100644index 00000000000..cc620a740cf--- /dev/null+++ b/test/metabase/search/engine_test.clj@@ -0,0 +1,114 @@+(ns metabase.search.engine-test+ (:require+ [clojure.test :refer :all]+ [metabase.search.engine :as search.engine]+ ;; Loaded for side effects: registers the engine implementations.+ [metabase.search.init]+ [metabase.search.settings :as search.settings]+ [metabase.search.task.search-index :as task.search-index]+ [metabase.task.core :as task]+ [metabase.test :as mt]+ [metabase.test.util :as tu]))++(def ^:private all-engines+ #{:search.engine/semantic :search.engine/appdb :search.engine/in-place})++(defmacro ^:private with-engines+ "Run `body` with the given engine configuration, bypassing the real capability checks and settings.+ `supported` is the set of capable engines, `configured` the search-engine setting value, and `additional`+ the additional-search-engines setting value."+ [{:keys [supported configured additional]} & body]+ ;; with-redefs is required: supported-engine? is a multimethod, which with-dynamic-fn-redefs cannot proxy.+ #_{:clj-kondo/ignore [:metabase/prefer-with-dynamic-fn-redefs]}+ `(with-redefs [search.engine/supported-engine? ~supported+ search.engine/known-engine? all-engines+ search.settings/configured-search-engine (constantly ~configured)+ search.settings/additional-search-engines (constantly ~additional)]+ ~@body))++(deftest default-engine-resolution-test+ (testing "the first supported engine in the precedence wins"+ (with-engines {:supported all-engines}+ (is (= :search.engine/semantic (search.engine/default-engine))))+ (with-engines {:supported #{:search.engine/appdb :search.engine/in-place}}+ (is (= :search.engine/appdb (search.engine/default-engine))))+ (with-engines {:supported #{:search.engine/in-place}}+ (is (= :search.engine/in-place (search.engine/default-engine)))))+ (testing "a configured engine overrides the precedence"+ (with-engines {:supported all-engines :configured :appdb}+ (is (= :search.engine/appdb (search.engine/default-engine))))+ (with-engines {:supported all-engines :configured :in-place}+ (is (= :search.engine/in-place (search.engine/default-engine)))))+ (testing "a configured engine that is not supported falls back to the precedence"+ (with-engines {:supported #{:search.engine/appdb :search.engine/in-place} :configured :semantic}+ (is (= :search.engine/appdb (search.engine/default-engine)))))+ (testing "an unknown configured engine is ignored rather than breaking resolution"+ (with-engines {:supported all-engines :configured :elastic}+ (is (= :search.engine/semantic (search.engine/default-engine)))))+ (testing "a legacy engine name is canonicalized"+ (with-engines {:supported all-engines :configured :fulltext}+ (is (= :search.engine/appdb (search.engine/default-engine))))))++(deftest search-engine-setting-test+ (testing "the setting computes the resolved engine when no value is configured"+ (with-engines {:supported #{:search.engine/appdb :search.engine/in-place}}+ (is (= :appdb (search.settings/search-engine))))+ (with-engines {:supported all-engines}+ (is (= :semantic (search.settings/search-engine)))))+ (testing "a configured value is returned as-is"+ (with-engines {:supported all-engines :configured :in-place}+ (is (= :in-place (search.settings/search-engine)))))+ (testing "an override that cannot be honored reports the engine actually serving"+ (with-engines {:supported #{:search.engine/appdb :search.engine/in-place} :configured :semantic}+ (is (= :appdb (search.settings/search-engine))))))++(deftest supported-engines-test+ (testing "the configured engine leads, followed by the precedence"+ (with-engines {:supported all-engines :configured :in-place}+ (is (= [:search.engine/in-place :search.engine/semantic :search.engine/appdb]+ (search.engine/supported-engines))))))++(deftest active-engines-test+ (testing "a semantic default activates appdb, its dependency"+ (with-engines {:supported all-engines}+ (is (= [:search.engine/semantic :search.engine/appdb] (search.engine/active-engines)))))+ (testing "an appdb default activates only appdb"+ (with-engines {:supported all-engines :configured :appdb}+ (is (= [:search.engine/appdb] (search.engine/active-engines)))))+ (testing "an in-place default activates nothing"+ (with-engines {:supported #{:search.engine/in-place}}+ (is (= [] (search.engine/active-engines)))))+ (testing "additional engines are activated alongside the default, with their dependencies"+ (with-engines {:supported all-engines :configured :appdb :additional ["semantic"]}+ (is (= [:search.engine/appdb :search.engine/semantic] (search.engine/active-engines))))+ (with-engines {:supported all-engines :configured :in-place :additional ["semantic"]}+ (is (= [:search.engine/semantic :search.engine/appdb] (search.engine/active-engines)))))+ (testing "unknown or unsupported additional engines are ignored"+ (with-engines {:supported #{:search.engine/appdb :search.engine/in-place}+ :configured :appdb+ :additional ["semantic" "elastic"]}+ (is (= [:search.engine/appdb] (search.engine/active-engines)))))+ (testing "legacy engine names in additional engines are canonicalized"+ (with-engines {:supported all-engines :configured :in-place :additional ["fulltext"]}+ (is (= [:search.engine/appdb] (search.engine/active-engines)))))+ (testing "additional engines tolerate csv whitespace, blanks, and qualified names"+ (with-engines {:supported all-engines :configured :appdb :additional [" semantic" " " "search.engine/appdb"]}+ (is (= [:search.engine/appdb :search.engine/semantic] (search.engine/active-engines))))))++(deftest additional-search-engines-setter-test+ (let [triggered (atom [])]+ (mt/with-dynamic-fn-redefs [task/job-exists? (constantly true)+ task/trigger-now! (fn [k] (swap! triggered conj k))+ ;; Track the setting so the setter sees the newly added engine.+ search.engine/active-engines (fn []+ (map #(keyword "search.engine" %)+ (search.settings/additional-search-engines)))]+ (testing "activating an engine triggers the search index init task"+ (mt/with-temporary-setting-values [additional-search-engines ["semantic"]]+ ;; The setter triggers from a post-commit future.+ (is (tu/poll-until 10000 (some #{task.search-index/init-job-key} @triggered)))+ (testing "but nothing triggers when no engine is newly active"+ (reset! triggered [])+ ;; Deterministic negative: call the trigger check directly with the current engines as the baseline.+ (task.search-index/trigger-init-for-newly-active-engines! (set (search.engine/active-engines)))+ (is (empty? @triggered))))))))diff --git a/test/metabase/search/filter_test.clj b/test/metabase/search/filter_test.cljindex 8d529cd4feb..87e4ccd8adf 100644--- a/test/metabase/search/filter_test.clj+++ b/test/metabase/search/filter_test.clj@@ -14,7 +14,7 @@(use-fixtures :once (fixtures/initialize :db))(use-fixtures :each (fn [thunk] (binding [search.ingestion/*force-sync* true]- (search.tu/with-new-search-if-available-otherwise-legacy (thunk)))))+ (search.tu/with-appdb-search-if-available-otherwise-legacy (thunk)))))(defn- filter-keys [](remove #{:ids} (map :context-key (vals search.config/filters))))diff --git a/test/metabase/search/ingestion_test.clj b/test/metabase/search/ingestion_test.cljindex 3f2ed8b55d4..776172aaae7 100644--- a/test/metabase/search/ingestion_test.clj+++ b/test/metabase/search/ingestion_test.clj@@ -206,7 +206,7 @@(deftest curation-signals-surfaced-in-results-test(testing "curated (all models) and table data_layer ride through legacy_input to appdb search results,so Metabot can render them (BOT-1570) — not just stored in the filtering column"- (search.tu/with-new-search-if-available-without-fallback+ (search.tu/with-appdb-search-if-available-without-fallback(mt/with-temp [:model/Database {db-id :id} {}:model/Table _ {:db_id db-id:name "Curatedgoldtbl"diff --git a/test/metabase/search/test_util.clj b/test/metabase/search/test_util.cljindex 1fbff5d6b2e..7e1d6f69a1b 100644--- a/test/metabase/search/test_util.clj+++ b/test/metabase/search/test_util.clj@@ -31,43 +31,54 @@~@body))))#_{:clj-kondo/ignore [:metabase/test-helpers-use-non-thread-safe-functions]}-(defmacro with-new-search-if-available*+(defmacro with-appdb-search-if-available*"Create a temporary index table for the duration of the body."[& body]- `(mt/with-dynamic-fn-redefs [search.engine/default-engine (constantly :search.engine/appdb)]+ `(mt/with-dynamic-fn-redefs [search.engine/default-engine (constantly :search.engine/appdb)+ ;; active-engines derives from the default engine, so pinning the default+ ;; would deactivate semantic; keep it active (and indexed) when supported.+ search.engine/active-engines (fn []+ (cond-> [:search.engine/appdb]+ (search.engine/supported-engine? :search.engine/semantic)+ (conj :search.engine/semantic)))](with-temp-index-table(search/reindex! {:async? false :in-place? true})~@body)))#_{:clj-kondo/ignore [:metabase/test-helpers-use-non-thread-safe-functions]}-(defmacro with-new-search-if-available-otherwise-legacy+(defmacro with-appdb-search-if-available-otherwise-legacy"Create a temporary index table for the duration of the body."[& body]`(if (search/supports-index?)- (with-new-search-if-available* ~@body)+ (with-appdb-search-if-available* ~@body)~@body))-(defmacro with-new-search-if-available-without-fallback+(defmacro with-appdb-search-if-available-without-fallback"Create a temporary index table for the duration of the body.Only runs if the appdb search engine is supported."[& body]`(when (search.engine/supported-engine? :search.engine/appdb)- (with-new-search-if-available* ~@body)))+ (with-appdb-search-if-available* ~@body)))(defmacro with-legacy-search"Ensure legacy search, which doesn't require an index, is used.Semantic queries go to :search.engine/semantic and keyword queries fall back to :search.engine/in-place."[& body]`(mt/with-dynamic-fn-redefs [search.engine/default-engine (constantly :search.engine/in-place)- search.engine/supported-engines (constantly [:search.engine/semantic :search.engine/in-place])]+ search.engine/supported-engines (constantly [:search.engine/semantic :search.engine/in-place])+ ;; active-engines derives from the default engine; semantic must stay active+ ;; when supported for semantic queries to reach it.+ search.engine/active-engines (fn []+ (when (search.engine/supported-engine? :search.engine/semantic)+ [:search.engine/semantic]))]~@body))-(defmacro with-new-search-and-legacy-search+(defmacro with-appdb-search-and-legacy-search"Run the body twice, once with the legacy search engine, and once with the appdb search engine."[& body]`(do(with-legacy-search ~@body)- (with-new-search-if-available-without-fallback ~@body)))+ (with-appdb-search-if-available-without-fallback ~@body)))(defmacro with-index-disabled"Skip any index maintenance during this test."diff --git a/test/metabase/transforms_rest/api/transform_test.clj b/test/metabase/transforms_rest/api/transform_test.cljindex 7d0230ff9f0..6926a9887ff 100644--- a/test/metabase/transforms_rest/api/transform_test.clj+++ b/test/metabase/transforms_rest/api/transform_test.clj@@ -1876,7 +1876,7 @@:schema (get-test-schema):name (str "target_" (u/generate-nano-id)):database (mt/id)}}]- (search.tu/with-new-search-and-legacy-search+ (search.tu/with-appdb-search-and-legacy-search(let [transform-ids (search-transform-ids search-term)](is (contains? transform-ids query-id))(is (not (contains? transform-ids python-id)))))))))))@@ -1894,7 +1894,7 @@:target {:type "table":schema (get-test-schema):name (str "target_" (u/generate-nano-id))}}]- (search.tu/with-new-search-and-legacy-search+ (search.tu/with-appdb-search-and-legacy-search(let [results (mt/user-http-request :rasta :get 200 "search" :q search-term :models "transform")ids (set (map :id (:data results)))](is (empty? ids))@@ -1921,7 +1921,7 @@:target {:type "table":schema (get-test-schema):name (str "target_" (u/generate-nano-id))}}]- (search.tu/with-new-search-and-legacy-search+ (search.tu/with-appdb-search-and-legacy-search(is (= #{native-id mbql-id}(search-transform-ids search-term))))))))))