The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Experimental options for improving Metabot Search (#65691)
Chris Truterc75dd94d0238
diff --git a/.github/workflows/semantic-search.yml b/.github/workflows/semantic-search.ymlindex 942bab24f3f..7daa172263b 100644--- a/.github/workflows/semantic-search.yml+++ b/.github/workflows/semantic-search.yml@@ -27,7 +27,7 @@ jobs:- name: Semantic Search Testsbuild-static-viz: falsetest-args: >-- :only '"enterprise/backend/test/metabase_enterprise/semantic_search"'+ :only '["enterprise/backend/test/metabase_enterprise/semantic_search" "enterprise/backend/test/metabase_enterprise/metabot_v3/tools/semantic_search_test.clj"]'exclude-tag: ':mb/driver-tests'services:@@ -85,7 +85,7 @@ jobs:- name: Semantic Search Testsbuild-static-viz: falsetest-args: >-- :only '"enterprise/backend/test/metabase_enterprise/semantic_search"'+ :only '["enterprise/backend/test/metabase_enterprise/semantic_search" "enterprise/backend/test/metabase_enterprise/metabot_v3/tools/semantic_search_test.clj"]'exclude-tag: ':mb/driver-tests'services:@@ -157,7 +157,7 @@ jobs:- name: Semantic Search Testsbuild-static-viz: falsetest-args: >-- :only '"enterprise/backend/test/metabase_enterprise/semantic_search"'+ :only '["enterprise/backend/test/metabase_enterprise/semantic_search" "enterprise/backend/test/metabase_enterprise/metabot_v3/tools/semantic_search_test.clj"]'exclude-tag: ':mb/driver-tests'services:@@ -229,7 +229,7 @@ jobs:- name: Semantic Search Testsbuild-static-viz: falsetest-args: >-- :only '"enterprise/backend/test/metabase_enterprise/semantic_search"'+ :only '["enterprise/backend/test/metabase_enterprise/semantic_search" "enterprise/backend/test/metabase_enterprise/metabot_v3/tools/semantic_search_test.clj"]'exclude-tag: ':mb/driver-tests'services:diff --git a/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/api.clj b/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/api.cljindex 45c3091c3b3..ed92936fc06 100644--- a/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/api.clj+++ b/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/api.clj@@ -1040,18 +1040,23 @@(assoc :conversation_id conversation_id))(metabot-v3.context/log :llm.log/be->llm))))+(def ^:private experimental-flags [:unified_disjunct_querying :separate_disjunct_querying])+(mr/def ::search-arguments[:and- [:map- [:term_queries {:optional true} [:maybe [:sequential :string]]]- [:semantic_queries {:optional true} [:maybe [:sequential :string]]]- [:entity_types {:optional true} [:maybe [:sequential [:enum "table" "model" "question" "dashboard" "metric" "database" "transform"]]]]- [:database_id {:optional true} [:maybe :int]]- [:created_at {:optional true} [:maybe ms/NonBlankString]]- [:last_edited_at {:optional true} [:maybe ms/NonBlankString]]- [:search_native_query {:optional true, :default false} [:maybe :boolean]]- [:limit {:optional true, :default 50} [:and :int [:fn #(<= 1 % 100)]]]- [:weights {:optional true} [:map-of :keyword number?]]]+ (into+ [:map+ [:term_queries {:optional true} [:maybe [:sequential :string]]]+ [:semantic_queries {:optional true} [:maybe [:sequential :string]]]+ [:entity_types {:optional true} [:maybe [:sequential [:enum "table" "model" "question" "dashboard" "metric" "database" "transform"]]]]+ [:database_id {:optional true} [:maybe :int]]+ [:created_at {:optional true} [:maybe ms/NonBlankString]]+ [:last_edited_at {:optional true} [:maybe ms/NonBlankString]]+ [:search_native_query {:optional true, :default false} [:maybe :boolean]]+ [:limit {:optional true, :default 50} [:and :int [:fn #(<= 1 % 100)]]]+ [:weights {:optional true} [:map-of :keyword number?]]]+ (for [f experimental-flags]+ [f {:optional true} [:maybe :boolean]]))[:map {:encode/tool-api-request#(update-keys % (comp keyword u/->kebab-case-en))}]])@@ -1088,6 +1093,9 @@(try(let [options (mc/encode ::search-argumentsarguments (mtx/transformer {:name :tool-api-request}))+ options (-> (apply dissoc options experimental-flags)+ (assoc :experimental-opts (zipmap (map u/kebab->snake experimental-flags)+ (map options experimental-flags))))metabot-id (:metabot-v3/metabot-id request)results (metabot-v3.tools.search/search(assoc options :metabot-id metabot-id))diff --git a/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/search.clj b/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/search.cljindex 426bbd6ce10..47099a43fb4 100644--- a/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/search.clj+++ b/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/search.clj@@ -6,6 +6,8 @@[metabase.api.common :as api][metabase.permissions.core :as perms][metabase.search.core :as search]+ [metabase.search.engine :as search.engine]+ [metabase.util :as u][metabase.util.log :as log][toucan2.core :as t2]))@@ -96,88 +98,120 @@([result-lists](reciprocal-rank-fusion result-lists 60))([result-lists k]- (let [rrf-results (reduce- (fn [acc-map result-list]- (reduce-kv- (fn [acc rank search-result]- (let [id (search-result-id search-result)- rrf-score (/ 1.0 (+ k (inc rank)))]- (if (contains? acc id)- (update-in acc [id :rrf] + rrf-score)- (assoc acc id {:search-result search-result- :rrf rrf-score}))))- acc-map- (vec result-list)))- {}- result-lists)]- (->> rrf-results- vals- (sort-by :rrf >)- (map :search-result)))))+ ;; Remove empty result lists, as they're common, and can save a lot of work.+ (let [result-lists (keep seq result-lists)]+ (if (<= (count result-lists) 1)+ (first result-lists)+ (let [rrf-results (reduce+ (fn [acc-map result-list]+ (reduce-kv+ (fn [acc rank search-result]+ (let [id (search-result-id search-result)+ rrf-score (/ 1.0 (+ k (inc rank)))]+ (if (contains? acc id)+ (update-in acc [id :rrf] + rrf-score)+ (assoc acc id {:search-result search-result+ :rrf rrf-score}))))+ acc-map+ (vec result-list)))+ {}+ result-lists)]+ (->> rrf-results+ vals+ (sort-by :rrf >)+ (map :search-result)))))))++(defn- join-results-by-rrf+ "Execute multiple search queries in parallel and combine results using Reciprocal Rank Fusion.+ Items appearing in multiple result lists are boosted in the final ranking.+ May return more results than requested limit."+ [search-fn search-engine all-queries]+ ;; Zero queries case is handled nicely by the >1 branch+ (if (= 1 (count all-queries))+ (search-fn (first all-queries) search-engine)+ ;; Create futures for parallel execution+ (let [futures (mapv #(future (search-fn % search-engine)) all-queries)+ result-lists (mapv deref futures)]+ (reciprocal-rank-fusion result-lists))))(defn search"Search for data sources (tables, models, cards, dashboards, metrics, transforms) in Metabase.Abstracted from the API endpoint logic."[{:keys [term-queries semantic-queries database-id created-at last-edited-at- entity-types limit metabot-id search-native-query weights]}]+ entity-types limit metabot-id search-native-query weights]+ ;; Temporary parameters we're experimenting with (missing unless explicitly sent to API)+ {:keys [unified-disjunct-querying split-semantic-terms]} :experimental-opts}](log/infof "[METABOT-SEARCH] Starting search with params: %s"- {:term-queries term-queries- :semantic-queries semantic-queries- :database-id database-id- :created-at created-at- :last-edited-at last-edited-at- :entity-types entity-types- :limit limit- :metabot-id metabot-id+ {:term-queries term-queries+ :semantic-queries semantic-queries+ :database-id database-id+ :created-at created-at+ :last-edited-at last-edited-at+ :entity-types entity-types+ :limit limit+ :metabot-id metabot-id:search-native-query search-native-query- :weights weights})- (let [search-models (if (seq entity-types)- (set (distinct (keep entity-type->search-model entity-types)))- metabot-search-models)- _ (log/infof "[METABOT-SEARCH] Converted entity-types %s to search-models %s" entity-types search-models)- all-queries (distinct (concat (or term-queries []) (or semantic-queries [])))- metabot (t2/select-one :model/Metabot :entity_id (get-in metabot-v3.config/metabot-config [metabot-id :entity-id] metabot-id))- use-verified-content? (if metabot-id- (:use_verified_content metabot)- false)- limit (or limit 50)- search-fn (fn [query]- (let [search-context (search/search-context- (cond->- {:search-string query- :models search-models- :table-db-id database-id- :created-at created-at- :last-edited-at last-edited-at- :current-user-id api/*current-user-id*- :is-impersonated-user? (perms/impersonated-user?)- :is-sandboxed-user? (perms/sandboxed-user?)- :is-superuser? api/*is-superuser?*- :current-user-perms @api/*current-user-permissions-set*- :filter-items-in-personal-collection "exclude-others"- :context :metabot- :archived false- :limit limit- :offset 0}- ;; Don't include search-native-query key if nil so that we don't- ;; inadvertently filter out search models that don't support it- search-native-query- (assoc :search-native-query (boolean search-native-query))- use-verified-content?- (assoc :verified true)- weights- (assoc :weights weights)))- _ (log/infof "[METABOT-SEARCH] Search context models for query '%s': %s"- query (:models search-context))- search-results (search/search search-context)- data (:data search-results)- result-models (frequencies (map :model data))]- (log/infof "[METABOT-SEARCH] Query '%s' returned entity types: %s" query result-models)- data))- ;; Create futures for parallel execution- futures (mapv #(future (search-fn %)) all-queries)- result-lists (mapv deref futures)- fused-results (take limit (reciprocal-rank-fusion result-lists))]+ :weights weights})+ (let [search-models (if (seq entity-types)+ (set (distinct (keep entity-type->search-model entity-types)))+ metabot-search-models)+ _ (log/infof "[METABOT-SEARCH] Converted entity-types %s to search-models %s" entity-types search-models)+ metabot (t2/select-one :model/Metabot :entity_id (get-in metabot-v3.config/metabot-config [metabot-id :entity-id] metabot-id))+ use-verified? (if metabot-id+ (:use_verified_content metabot)+ false)+ limit (or limit 50)+ search-fn (fn [search-string search-engine]+ (let [search-context (search/search-context+ (cond-> {:search-string search-string+ :models search-models+ :table-db-id database-id+ :created-at created-at+ :last-edited-at last-edited-at+ :current-user-id api/*current-user-id*+ :is-impersonated-user? (perms/impersonated-user?)+ :is-sandboxed-user? (perms/sandboxed-user?)+ :is-superuser? api/*is-superuser?*+ :current-user-perms @api/*current-user-permissions-set*+ :filter-items-in-personal-collection "exclude-others"+ :context :metabot+ :archived false+ :limit limit+ :offset 0}+ ;; Don't include search-native-query key if nil so that we don't+ ;; inadvertently filter out search models that don't support it+ search-native-query+ (assoc :search-native-query (boolean search-native-query))+ use-verified?+ (assoc :verified true)+ weights+ (assoc :weights weights)+ search-engine+ (assoc :search-engine (name search-engine))))+ _ (log/infof "[METABOT-SEARCH] Search context models for query '%s': %s"+ search-string (:models search-context))+ search-results (search/search search-context)+ data (:data search-results)+ result-models (frequencies (map :model data))]+ (log/infof "[METABOT-SEARCH] Query '%s' returned entity types: %s" search-string result-models)+ data))+ search-fn* (fn [search-engine queries]+ (let [queries (if unified-disjunct-querying (search.engine/disjunction search-engine queries) queries)]+ (join-results-by-rrf search-fn search-engine queries)))+ ;; NOTE: if we add more semantic engines, e.g. 3rd party vector dbs, we'll need to make this more maintainable+ semantic? #{:search.engine/semantic}+ semantic-engine (u/seek semantic? (search.engine/active-engines))+ fallback-engine (when semantic-engine+ (u/seek (comp not semantic?) (search.engine/supported-engines)))+ fused-results (if (and split-semantic-terms semantic-engine)+ ;; Perform semantic and non-semantic search respectively, then fuse results.+ (reciprocal-rank-fusion+ (map (fn [[engine queries]] (when (seq queries) (search-fn* engine queries)))+ {semantic-engine semantic-queries+ fallback-engine term-queries}))+ ;; Search for all the terms on equal footing, using the default engine.+ (search-fn* nil (distinct (concat term-queries semantic-queries))))](->> fused-results+ (take limit)(map postprocess-search-result)enrich-with-collection-descriptions)))diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/db/datasource.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/db/datasource.cljindex 27408da1dd8..b6d7872fc92 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/db/datasource.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/db/datasource.clj@@ -18,7 +18,7 @@(env :mb-pgvector-db-url));; See metabase.app-db.connection-pool-setup for more details on these properties-;; TODO: not sure if we need the MetabaseConnectionCustomizer like in the app DB connection pool seutp+;; TODO: not sure if we need the MetabaseConnectionCustomizer like in the app DB connection pool setup(def ^:private semantic-search-connection-pool-props"Connection pool properties for semantic search database using c3p0."{"idleConnectionTestPeriod" 60diff --git a/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/semantic_search_test.clj b/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/semantic_search_test.cljnew file mode 100644index 00000000000..3f7c83d9913--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/semantic_search_test.clj@@ -0,0 +1,97 @@+(ns metabase-enterprise.metabot-v3.tools.semantic-search-test+ "Semantic search tests specific to the Metabot search tool integration."+ (:require+ [clojure.test :refer :all]+ [metabase-enterprise.metabot-v3.tools.search :as search]+ [metabase-enterprise.semantic-search.test-util :as semantic.tu]+ [metabase.search.engine :as search.engine]+ [metabase.search.ingestion :as search.ingestion]+ [metabase.search.test-util :as search.tu]+ [metabase.test :as mt]))++(use-fixtures :once #'semantic.tu/once-fixture)++(defmacro with-semantic-search-if-available! [mock-embeddings & body]+ `(mt/with-premium-features #{:semantic-search}+ (when (search.engine/supported-engine? :search.engine/semantic)+ (semantic.tu/with-mock-embeddings ~mock-embeddings+ (binding [search.ingestion/*disable-updates* false+ search.ingestion/*force-sync* true]+ ~@body)))))++(defmacro with-and-without-semantic-search! [mock-embeddings & body]+ `(do ~@body (with-semantic-search-if-available! ~mock-embeddings ~@body)))++;; Mock embeddings: similar vectors for semantic synonym pairs, orthogonal vectors for unrelated terms.+(def ^:private test-mock-embeddings+ {"belligerent" [1.0 0.0 0.0 0.0]+ "combative" [0.99 0.01 0.0 0.0]+ "bellicose" [0.0 1.0 0.0 0.0]+ "quarrelsome" [0.01 0.99 0.0 0.0]+ "quixotic" [0.0 0.0 1.0 0.0]+ "ancillary" [0.0 0.0 0.0 0.9]+ "adjunct" [0.0 0.0 0.0 1.0]+ "baseline" [0.5 0.5 0.0 0.0]})++(deftest split-keywords-only-test+ (testing "search returns only exact matches for keyword terms when {:split-semantic-terms true}, regardless of whether semantic search is enabled\n"+ (mt/with-test-user :rasta+ (semantic.tu/with-test-db! {:mode :mock-initialized}+ (with-and-without-semantic-search! test-mock-embeddings+ (search.tu/with-new-search-and-legacy-search+ (let [semantic-support? (search.engine/supported-engine? :search.engine/semantic)]+ ;; "belligerent" and "bellicose" are semantically similar to our search terms+ ;; ("combative", "quarrelsome") but should NOT match since we're only doing keyword search+ (mt/with-temp [:model/Dashboard {id-1 :id} {:name "belligerent"}+ :model/Dashboard {id-2 :id} {:name "bellicose"}+ ;; "baseline" will match via keyword/fulltext search+ :model/Dashboard {id-3 :id} {:name "baseline"}]+ (when semantic-support?+ (semantic.tu/index-all!))+ (doseq [unified-disjunct-querying [false true]]+ (testing (str "{unified-disjunct-querying " unified-disjunct-querying "}\n")+ (let [base-query {:term-queries ["combative" "quarrelsome" "baseline"]+ :semantic-queries []}+ test-entity? (comp #{id-1 id-2 id-3} :id)+ query (fn [unified-disjunct-querying]+ (->> (search/search (assoc base-query+ :experimental-opts+ {:unified-disjunct-querying unified-disjunct-querying+ :split-semantic-terms true}))+ (filter test-entity?)+ (map :name)))]+ (testing "Semantic results are not returned for keyword terms"+ (is (= #{"baseline"}+ (set (query unified-disjunct-querying))))))))))))))))++(deftest split-keyword-and-semantic-test+ (testing "search returns only exact matches for keyword terms when {:split-semantic-terms true}\n"+ (mt/with-test-user :rasta+ (semantic.tu/with-test-db! {:mode :mock-initialized}+ (with-semantic-search-if-available! test-mock-embeddings+ (search.tu/with-new-search-and-legacy-search+ ;; "belligerent" and "baseline" will match via keyword search (exact match in term-queries)+ ;; "ancillary" and "adjunct" will match via semantic search (similar embeddings)+ ;; "bellicose" and "quixotic" should NOT match (not in search terms)+ (mt/with-temp [:model/Dashboard {id-1 :id} {:name "belligerent"}+ :model/Dashboard {id-2 :id} {:name "bellicose"}+ :model/Dashboard {id-3 :id} {:name "ancillary"}+ :model/Dashboard {id-4 :id} {:name "adjunct"}+ :model/Dashboard {id-5 :id} {:name "quixotic"}+ :model/Dashboard {id-6 :id} {:name "baseline"}]+ (semantic.tu/index-all!)+ (doseq [unified-disjunct-querying [false true]]+ (testing (str "{unified-disjunct-querying " unified-disjunct-querying "}\n")+ (let [base-query {:term-queries ["baseline" "belligerent"]+ :semantic-queries ["ancillary"]}+ test-entity? (comp #{id-1 id-2 id-3 id-4 id-5 id-6} :id)+ query (fn [unified-disjunct-querying]+ (->> (search/search (assoc base-query+ :experimental-opts+ {:unified-disjunct-querying unified-disjunct-querying+ :split-semantic-terms true}))+ (filter test-entity?)+ (map :name)))]+ (testing "Semantic results are only returned for semantic terms"+ (is (= #{"baseline" "belligerent" "ancillary" "adjunct"}+ (set (query unified-disjunct-querying)))))))))))))))diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/test_util.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/test_util.cljindex 73fed24b551..1511be1ca88 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/test_util.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/test_util.clj@@ -228,16 +228,44 @@mock-documents-for-embeddingsbase-embedding-vectors)))+(def ^:dynamic *extra-mock-embeddings*+ "Dynamic var for test-specific mock embeddings. Merged with [[mock-embeddings]] in [[get-mock-embedding]]."+ nil)++(defn- parse-entity-name+ "Extract the entity name from embeddable text format.+ E.g., '[dashboard]\\nname: My Dashboard\\n...' -> 'My Dashboard'"+ [text]+ (second (re-find #"(?m)^name:\s*(.+)$" text)))+(defn get-mock-embedding- "Lookup the embedding for `text` in [[mock-embeddings]]."+ "Lookup the embedding for `text` in [[*extra-mock-embeddings*]] first, then [[mock-embeddings]].+ Lookups try both the raw text and the parsed entity name from embeddable text format."[text]- (get mock-embeddings text [0.01 0.02 0.03 0.04]))+ (let [entity-name (parse-entity-name text)]+ (or (get *extra-mock-embeddings* text)+ (get *extra-mock-embeddings* entity-name)+ (get mock-embeddings text)+ (get mock-embeddings entity-name)+ [0.01 0.02 0.03 0.04])))(defn get-mock-embeddings-batch"Lookup embeddings for multiple texts in [[mock-embeddings]]."[texts](mapv get-mock-embedding texts))+(defmacro with-mock-embeddings+ "Bind extra mock embeddings for the duration of `body`. The embeddings map should be+ a map from text strings to 4-dimensional vectors, e.g.:++ (with-mock-embeddings+ {\"belligerent\" [0.9 0.1 0.0 0.0]+ \"combative\" [0.91 0.11 0.01 0.01]} ; similar vector = semantic match+ ...)"+ [embeddings-map & body]+ `(binding [*extra-mock-embeddings* ~embeddings-map]+ ~@body))+;;;; mock provider(def mock-embedding-model@@ -446,7 +474,7 @@`(do-with-indexable-documents! (fn [] ~@body)))(defn index-all!- "Run indexer synchonously until we've exhausted polling all documents"+ "Run indexer synchronously until we've exhausted polling all documents"[](let [metadata-row {:indexer_last_poll Instant/EPOCH:indexer_last_seen Instant/EPOCH}diff --git a/enterprise/backend/test/metabase_enterprise/transforms/api/search_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/api/search_test.cljindex e717aca535e..bfb4e450a56 100644--- a/enterprise/backend/test/metabase_enterprise/transforms/api/search_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms/api/search_test.clj@@ -10,7 +10,7 @@(use-fixtures :each (fn [thunk](binding [search.ingestion/*force-sync* true]- (search.tu/with-new-search-if-available (thunk)))))+ (search.tu/with-new-search-if-available-otherwise-legacy (thunk)))))(deftest basic-search-test(testing "Transform can be found with search"diff --git a/src/metabase/search/appdb/core.clj b/src/metabase/search/appdb/core.cljindex 0bf603e6737..925b170fe09 100644--- a/src/metabase/search/appdb/core.clj+++ b/src/metabase/search/appdb/core.clj@@ -1,5 +1,6 @@(ns metabase.search.appdb.core(:require+ [clojure.string :as str][environ.core :as env][honey.sql.helpers :as sql.helpers][java-time.api :as t]@@ -49,6 +50,10 @@(#{"appdb" "semantic"} (some-> (search.settings/search-engine) name)))(supported-db? (mdb/db-type))))+(defmethod search.engine/disjunction :search.engine/appdb [_ terms]+ (when (seq terms)+ [(str/join " OR " (map #(str "(" % ")") terms))]))+(defn- parse-datetime [s](when s (OffsetDateTime/parse s)))diff --git a/src/metabase/search/core.clj b/src/metabase/search/core.cljindex 4d7d5a96c94..1df3c034744 100644--- a/src/metabase/search/core.clj+++ b/src/metabase/search/core.clj@@ -70,7 +70,7 @@(defmethod analytics/initial-value :metabase-search/engine-default[_ {:keys [engine]}]- (if (= engine (name (search.impl/default-engine))) 1 0))+ (if (= engine (name (search.engine/default-engine))) 1 0))(defmethod analytics/initial-value :metabase-search/engine-active[_ {:keys [engine]}]diff --git a/src/metabase/search/engine.clj b/src/metabase/search/engine.cljindex 9d3437aef39..e42f4ab24a0 100644--- a/src/metabase/search/engine.clj+++ b/src/metabase/search/engine.clj@@ -93,3 +93,20 @@[engine](let [registered? #(contains? (methods supported-engine?) %)](some registered? (cons engine (ancestors engine)))))++(defmulti disjunction+ "Given multiple terms to search for, reduce this to a search expression that matches any of them in a single search.+ If this is not possible, return a list of terms to be searched for separately."+ {:arglists '([search-engine terms])}+ (fn [search-engine _terms] search-engine))++(defn default-engine+ "In the absence of an explicit engine argument in a request, which engine should be used?"+ []+ ;; TODO (Chris 2025-11-07) It would be good to have a warning on start up whenever this is *not* what's configured.+ (first (supported-engines)))++(defmethod disjunction :default [_ terms] terms)++(defmethod disjunction nil [_ terms]+ (disjunction (default-engine) terms))diff --git a/src/metabase/search/impl.clj b/src/metabase/search/impl.cljindex 843d4e264d0..256f5d0ffb4 100644--- a/src/metabase/search/impl.clj+++ b/src/metabase/search/impl.clj@@ -215,12 +215,6 @@(not (zero? v))v))-(defn default-engine- "In the absence of an explicit engine argument in a request, which engine should be used?"- []- ;; TODO (Chris 2025-11-07) It would be good to have a warning on start up whenever this is *not* what's configured.- (first (search.engine/supported-engines)))-(defn- parse-engine [value](or (when-not (str/blank? value)(let [engine (keyword "search.engine" value)]@@ -233,12 +227,12 @@:elseengine)))- (default-engine)))+ (search.engine/default-engine)));; This forwarding is here for tests, we should clean those up.(defn- apply-default-engine [{:keys [search-engine] :as search-ctx}]- (let [default (default-engine)]+ (let [default (search.engine/default-engine)](when (= default search-engine)(throw (ex-info "Missing implementation for default search-engine" {:search-engine search-engine})))(log/debugf "Missing implementation for %s so instead using %s" search-engine default)diff --git a/src/metabase/search/in_place/legacy.clj b/src/metabase/search/in_place/legacy.cljindex 2b6e6d6c171..442a5dd6f3e 100644--- a/src/metabase/search/in_place/legacy.clj+++ b/src/metabase/search/in_place/legacy.clj@@ -31,6 +31,11 @@(defmethod search.engine/supported-engine? :search.engine/in-place [_]true)+(defmethod search.engine/disjunction :search.engine/in-place [_ terms]+ ;; The default composition is disjunction with this engine.+ (when (seq terms)+ [(str/join " " terms)]))+(defn search-model->revision-model"Return the appropriate revision model given a search model."[model]diff --git a/test/metabase/search/api_test.clj b/test/metabase/search/api_test.cljindex f5a8a92c999..66302f12b41 100644--- a/test/metabase/search/api_test.clj+++ b/test/metabase/search/api_test.clj@@ -37,7 +37,7 @@(def ^:private default-collection {:id false :name nil :authority_level nil :type nil})(use-fixtures :each (fn [thunk] (binding [search.ingestion/*force-sync* true]- (search.tu/with-new-search-if-available (thunk)))))+ (search.tu/with-new-search-if-available-otherwise-legacy (thunk)))))(def ^:private default-search-row{:archived falsediff --git a/test/metabase/search/filter_test.clj b/test/metabase/search/filter_test.cljindex 6f15bdd5511..dc5487213f5 100644--- a/test/metabase/search/filter_test.clj+++ b/test/metabase/search/filter_test.clj@@ -11,7 +11,7 @@[metabase.search.test-util :as search.tu]))(use-fixtures :each (fn [thunk] (binding [search.ingestion/*force-sync* true]- (search.tu/with-new-search-if-available (thunk)))))+ (search.tu/with-new-search-if-available-otherwise-legacy (thunk)))))(defn- filter-keys [](remove #{:ids} (map :context-key (vals search.config/filters))))diff --git a/test/metabase/search/test_util.clj b/test/metabase/search/test_util.cljindex e1081a8a8f1..f1a6fafd0cb 100644--- a/test/metabase/search/test_util.clj+++ b/test/metabase/search/test_util.clj@@ -31,22 +31,45 @@~@body))))#_{:clj-kondo/ignore [:metabase/test-helpers-use-non-thread-safe-functions]}-(defmacro with-new-search-if-available+(defmacro with-new-search-if-available*+ "Create a temporary index table for the duration of the body."+ [& body]+ `(mt/with-dynamic-fn-redefs [search.engine/default-engine (constantly :search.engine/appdb)]+ (with-temp-index-table+ (search/reindex! {:async? false :in-place? true})+ ~@body)))++#_{:clj-kondo/ignore [:metabase/test-helpers-use-non-thread-safe-functions]}+(defmacro with-new-search-if-available-otherwise-legacy"Create a temporary index table for the duration of the body."[& body]`(if (search/supports-index?)- (mt/with-dynamic-fn-redefs [search.impl/default-engine (constantly :search.engine/appdb)]- (with-temp-index-table- (search/reindex! {:async? false :in-place? true})- ~@body))+ (with-new-search-if-available* ~@body)~@body))+(defmacro with-new-search-if-available-without-fallback+ "Create a temporary index table for the duration of the body.+ Only runs if the appdb search engine is supported."+ [& body]+ `(when (search.engine/supported-engine? :search.engine/appdb)+ (with-new-search-if-available* ~@body)))+(defmacro with-legacy-search- "Ensure legacy search, which doesn't require an index, is used."+ "Ensure legacy search, which doesn't require an index, is used.+ When split-semantic-terms is enabled, semantic queries go to :search.engine/semantic+ and keyword queries fall back to :search.engine/in-place."[& body]- `(mt/with-dynamic-fn-redefs [search.impl/default-engine (constantly :search.engine/in-place)]+ `(mt/with-dynamic-fn-redefs [search.engine/default-engine (constantly :search.engine/in-place)+ search.engine/supported-engines (constantly [:search.engine/semantic :search.engine/in-place])]~@body))+(defmacro with-new-search-and-legacy-search+ "Run the body twice, once with the legacy search engine, and once with the appdb search engine."+ [& body]+ `(do+ (with-legacy-search ~@body)+ (with-new-search-if-available-without-fallback ~@body)))+(defmacro with-index-disabled"Skip any index maintenance during this test."[& body]