Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Experimental options for improving Metabot Search (#65691)

Chris Truterc75dd94d0238
diff --git a/.github/workflows/semantic-search.yml b/.github/workflows/semantic-search.yml
index 942bab24f3f..7daa172263b 100644
--- a/.github/workflows/semantic-search.yml
+++ b/.github/workflows/semantic-search.yml
@@ -27,7 +27,7 @@ jobs:
- name: Semantic Search Tests
build-static-viz: false
test-args: >-
- :only '"enterprise/backend/test/metabase_enterprise/semantic_search"'
+ :only '["enterprise/backend/test/metabase_enterprise/semantic_search" "enterprise/backend/test/metabase_enterprise/metabot_v3/tools/semantic_search_test.clj"]'
exclude-tag: ':mb/driver-tests'
services:
@@ -85,7 +85,7 @@ jobs:
- name: Semantic Search Tests
build-static-viz: false
test-args: >-
- :only '"enterprise/backend/test/metabase_enterprise/semantic_search"'
+ :only '["enterprise/backend/test/metabase_enterprise/semantic_search" "enterprise/backend/test/metabase_enterprise/metabot_v3/tools/semantic_search_test.clj"]'
exclude-tag: ':mb/driver-tests'
services:
@@ -157,7 +157,7 @@ jobs:
- name: Semantic Search Tests
build-static-viz: false
test-args: >-
- :only '"enterprise/backend/test/metabase_enterprise/semantic_search"'
+ :only '["enterprise/backend/test/metabase_enterprise/semantic_search" "enterprise/backend/test/metabase_enterprise/metabot_v3/tools/semantic_search_test.clj"]'
exclude-tag: ':mb/driver-tests'
services:
@@ -229,7 +229,7 @@ jobs:
- name: Semantic Search Tests
build-static-viz: false
test-args: >-
- :only '"enterprise/backend/test/metabase_enterprise/semantic_search"'
+ :only '["enterprise/backend/test/metabase_enterprise/semantic_search" "enterprise/backend/test/metabase_enterprise/metabot_v3/tools/semantic_search_test.clj"]'
exclude-tag: ':mb/driver-tests'
services:
diff --git a/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/api.clj b/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/api.clj
index 45c3091c3b3..ed92936fc06 100644
--- a/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/api.clj
@@ -1040,18 +1040,23 @@
(assoc :conversation_id conversation_id))
(metabot-v3.context/log :llm.log/be->llm))))
+(def ^:private experimental-flags [:unified_disjunct_querying :separate_disjunct_querying])
+
(mr/def ::search-arguments
[:and
- [:map
- [:term_queries {:optional true} [:maybe [:sequential :string]]]
- [:semantic_queries {:optional true} [:maybe [:sequential :string]]]
- [:entity_types {:optional true} [:maybe [:sequential [:enum "table" "model" "question" "dashboard" "metric" "database" "transform"]]]]
- [:database_id {:optional true} [:maybe :int]]
- [:created_at {:optional true} [:maybe ms/NonBlankString]]
- [:last_edited_at {:optional true} [:maybe ms/NonBlankString]]
- [:search_native_query {:optional true, :default false} [:maybe :boolean]]
- [:limit {:optional true, :default 50} [:and :int [:fn #(<= 1 % 100)]]]
- [:weights {:optional true} [:map-of :keyword number?]]]
+ (into
+ [:map
+ [:term_queries {:optional true} [:maybe [:sequential :string]]]
+ [:semantic_queries {:optional true} [:maybe [:sequential :string]]]
+ [:entity_types {:optional true} [:maybe [:sequential [:enum "table" "model" "question" "dashboard" "metric" "database" "transform"]]]]
+ [:database_id {:optional true} [:maybe :int]]
+ [:created_at {:optional true} [:maybe ms/NonBlankString]]
+ [:last_edited_at {:optional true} [:maybe ms/NonBlankString]]
+ [:search_native_query {:optional true, :default false} [:maybe :boolean]]
+ [:limit {:optional true, :default 50} [:and :int [:fn #(<= 1 % 100)]]]
+ [:weights {:optional true} [:map-of :keyword number?]]]
+ (for [f experimental-flags]
+ [f {:optional true} [:maybe :boolean]]))
[:map {:encode/tool-api-request
#(update-keys % (comp keyword u/->kebab-case-en))}]])
@@ -1088,6 +1093,9 @@
(try
(let [options (mc/encode ::search-arguments
arguments (mtx/transformer {:name :tool-api-request}))
+ options (-> (apply dissoc options experimental-flags)
+ (assoc :experimental-opts (zipmap (map u/kebab->snake experimental-flags)
+ (map options experimental-flags))))
metabot-id (:metabot-v3/metabot-id request)
results (metabot-v3.tools.search/search
(assoc options :metabot-id metabot-id))
diff --git a/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/search.clj b/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/search.clj
index 426bbd6ce10..47099a43fb4 100644
--- a/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/search.clj
+++ b/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/search.clj
@@ -6,6 +6,8 @@
[metabase.api.common :as api]
[metabase.permissions.core :as perms]
[metabase.search.core :as search]
+ [metabase.search.engine :as search.engine]
+ [metabase.util :as u]
[metabase.util.log :as log]
[toucan2.core :as t2]))
@@ -96,88 +98,120 @@
([result-lists]
(reciprocal-rank-fusion result-lists 60))
([result-lists k]
- (let [rrf-results (reduce
- (fn [acc-map result-list]
- (reduce-kv
- (fn [acc rank search-result]
- (let [id (search-result-id search-result)
- rrf-score (/ 1.0 (+ k (inc rank)))]
- (if (contains? acc id)
- (update-in acc [id :rrf] + rrf-score)
- (assoc acc id {:search-result search-result
- :rrf rrf-score}))))
- acc-map
- (vec result-list)))
- {}
- result-lists)]
- (->> rrf-results
- vals
- (sort-by :rrf >)
- (map :search-result)))))
+ ;; Remove empty result lists, as they're common, and can save a lot of work.
+ (let [result-lists (keep seq result-lists)]
+ (if (<= (count result-lists) 1)
+ (first result-lists)
+ (let [rrf-results (reduce
+ (fn [acc-map result-list]
+ (reduce-kv
+ (fn [acc rank search-result]
+ (let [id (search-result-id search-result)
+ rrf-score (/ 1.0 (+ k (inc rank)))]
+ (if (contains? acc id)
+ (update-in acc [id :rrf] + rrf-score)
+ (assoc acc id {:search-result search-result
+ :rrf rrf-score}))))
+ acc-map
+ (vec result-list)))
+ {}
+ result-lists)]
+ (->> rrf-results
+ vals
+ (sort-by :rrf >)
+ (map :search-result)))))))
+
+(defn- join-results-by-rrf
+ "Execute multiple search queries in parallel and combine results using Reciprocal Rank Fusion.
+ Items appearing in multiple result lists are boosted in the final ranking.
+ May return more results than requested limit."
+ [search-fn search-engine all-queries]
+ ;; Zero queries case is handled nicely by the >1 branch
+ (if (= 1 (count all-queries))
+ (search-fn (first all-queries) search-engine)
+ ;; Create futures for parallel execution
+ (let [futures (mapv #(future (search-fn % search-engine)) all-queries)
+ result-lists (mapv deref futures)]
+ (reciprocal-rank-fusion result-lists))))
(defn search
"Search for data sources (tables, models, cards, dashboards, metrics, transforms) in Metabase.
Abstracted from the API endpoint logic."
[{:keys [term-queries semantic-queries database-id created-at last-edited-at
- entity-types limit metabot-id search-native-query weights]}]
+ entity-types limit metabot-id search-native-query weights]
+ ;; Temporary parameters we're experimenting with (missing unless explicitly sent to API)
+ {:keys [unified-disjunct-querying split-semantic-terms]} :experimental-opts}]
(log/infof "[METABOT-SEARCH] Starting search with params: %s"
- {:term-queries term-queries
- :semantic-queries semantic-queries
- :database-id database-id
- :created-at created-at
- :last-edited-at last-edited-at
- :entity-types entity-types
- :limit limit
- :metabot-id metabot-id
+ {:term-queries term-queries
+ :semantic-queries semantic-queries
+ :database-id database-id
+ :created-at created-at
+ :last-edited-at last-edited-at
+ :entity-types entity-types
+ :limit limit
+ :metabot-id metabot-id
:search-native-query search-native-query
- :weights weights})
- (let [search-models (if (seq entity-types)
- (set (distinct (keep entity-type->search-model entity-types)))
- metabot-search-models)
- _ (log/infof "[METABOT-SEARCH] Converted entity-types %s to search-models %s" entity-types search-models)
- all-queries (distinct (concat (or term-queries []) (or semantic-queries [])))
- metabot (t2/select-one :model/Metabot :entity_id (get-in metabot-v3.config/metabot-config [metabot-id :entity-id] metabot-id))
- use-verified-content? (if metabot-id
- (:use_verified_content metabot)
- false)
- limit (or limit 50)
- search-fn (fn [query]
- (let [search-context (search/search-context
- (cond->
- {:search-string query
- :models search-models
- :table-db-id database-id
- :created-at created-at
- :last-edited-at last-edited-at
- :current-user-id api/*current-user-id*
- :is-impersonated-user? (perms/impersonated-user?)
- :is-sandboxed-user? (perms/sandboxed-user?)
- :is-superuser? api/*is-superuser?*
- :current-user-perms @api/*current-user-permissions-set*
- :filter-items-in-personal-collection "exclude-others"
- :context :metabot
- :archived false
- :limit limit
- :offset 0}
- ;; Don't include search-native-query key if nil so that we don't
- ;; inadvertently filter out search models that don't support it
- search-native-query
- (assoc :search-native-query (boolean search-native-query))
- use-verified-content?
- (assoc :verified true)
- weights
- (assoc :weights weights)))
- _ (log/infof "[METABOT-SEARCH] Search context models for query '%s': %s"
- query (:models search-context))
- search-results (search/search search-context)
- data (:data search-results)
- result-models (frequencies (map :model data))]
- (log/infof "[METABOT-SEARCH] Query '%s' returned entity types: %s" query result-models)
- data))
- ;; Create futures for parallel execution
- futures (mapv #(future (search-fn %)) all-queries)
- result-lists (mapv deref futures)
- fused-results (take limit (reciprocal-rank-fusion result-lists))]
+ :weights weights})
+ (let [search-models (if (seq entity-types)
+ (set (distinct (keep entity-type->search-model entity-types)))
+ metabot-search-models)
+ _ (log/infof "[METABOT-SEARCH] Converted entity-types %s to search-models %s" entity-types search-models)
+ metabot (t2/select-one :model/Metabot :entity_id (get-in metabot-v3.config/metabot-config [metabot-id :entity-id] metabot-id))
+ use-verified? (if metabot-id
+ (:use_verified_content metabot)
+ false)
+ limit (or limit 50)
+ search-fn (fn [search-string search-engine]
+ (let [search-context (search/search-context
+ (cond-> {:search-string search-string
+ :models search-models
+ :table-db-id database-id
+ :created-at created-at
+ :last-edited-at last-edited-at
+ :current-user-id api/*current-user-id*
+ :is-impersonated-user? (perms/impersonated-user?)
+ :is-sandboxed-user? (perms/sandboxed-user?)
+ :is-superuser? api/*is-superuser?*
+ :current-user-perms @api/*current-user-permissions-set*
+ :filter-items-in-personal-collection "exclude-others"
+ :context :metabot
+ :archived false
+ :limit limit
+ :offset 0}
+ ;; Don't include search-native-query key if nil so that we don't
+ ;; inadvertently filter out search models that don't support it
+ search-native-query
+ (assoc :search-native-query (boolean search-native-query))
+ use-verified?
+ (assoc :verified true)
+ weights
+ (assoc :weights weights)
+ search-engine
+ (assoc :search-engine (name search-engine))))
+ _ (log/infof "[METABOT-SEARCH] Search context models for query '%s': %s"
+ search-string (:models search-context))
+ search-results (search/search search-context)
+ data (:data search-results)
+ result-models (frequencies (map :model data))]
+ (log/infof "[METABOT-SEARCH] Query '%s' returned entity types: %s" search-string result-models)
+ data))
+ search-fn* (fn [search-engine queries]
+ (let [queries (if unified-disjunct-querying (search.engine/disjunction search-engine queries) queries)]
+ (join-results-by-rrf search-fn search-engine queries)))
+ ;; NOTE: if we add more semantic engines, e.g. 3rd party vector dbs, we'll need to make this more maintainable
+ semantic? #{:search.engine/semantic}
+ semantic-engine (u/seek semantic? (search.engine/active-engines))
+ fallback-engine (when semantic-engine
+ (u/seek (comp not semantic?) (search.engine/supported-engines)))
+ fused-results (if (and split-semantic-terms semantic-engine)
+ ;; Perform semantic and non-semantic search respectively, then fuse results.
+ (reciprocal-rank-fusion
+ (map (fn [[engine queries]] (when (seq queries) (search-fn* engine queries)))
+ {semantic-engine semantic-queries
+ fallback-engine term-queries}))
+ ;; Search for all the terms on equal footing, using the default engine.
+ (search-fn* nil (distinct (concat term-queries semantic-queries))))]
(->> fused-results
+ (take limit)
(map postprocess-search-result)
enrich-with-collection-descriptions)))
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/db/datasource.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/db/datasource.clj
index 27408da1dd8..b6d7872fc92 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/db/datasource.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/db/datasource.clj
@@ -18,7 +18,7 @@
(env :mb-pgvector-db-url))
;; See metabase.app-db.connection-pool-setup for more details on these properties
-;; TODO: not sure if we need the MetabaseConnectionCustomizer like in the app DB connection pool seutp
+;; TODO: not sure if we need the MetabaseConnectionCustomizer like in the app DB connection pool setup
(def ^:private semantic-search-connection-pool-props
"Connection pool properties for semantic search database using c3p0."
{"idleConnectionTestPeriod" 60
diff --git a/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/semantic_search_test.clj b/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/semantic_search_test.clj
new file mode 100644
index 00000000000..3f7c83d9913
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/semantic_search_test.clj
@@ -0,0 +1,97 @@
+(ns metabase-enterprise.metabot-v3.tools.semantic-search-test
+ "Semantic search tests specific to the Metabot search tool integration."
+ (:require
+ [clojure.test :refer :all]
+ [metabase-enterprise.metabot-v3.tools.search :as search]
+ [metabase-enterprise.semantic-search.test-util :as semantic.tu]
+ [metabase.search.engine :as search.engine]
+ [metabase.search.ingestion :as search.ingestion]
+ [metabase.search.test-util :as search.tu]
+ [metabase.test :as mt]))
+
+(use-fixtures :once #'semantic.tu/once-fixture)
+
+(defmacro with-semantic-search-if-available! [mock-embeddings & body]
+ `(mt/with-premium-features #{:semantic-search}
+ (when (search.engine/supported-engine? :search.engine/semantic)
+ (semantic.tu/with-mock-embeddings ~mock-embeddings
+ (binding [search.ingestion/*disable-updates* false
+ search.ingestion/*force-sync* true]
+ ~@body)))))
+
+(defmacro with-and-without-semantic-search! [mock-embeddings & body]
+ `(do ~@body (with-semantic-search-if-available! ~mock-embeddings ~@body)))
+
+;; Mock embeddings: similar vectors for semantic synonym pairs, orthogonal vectors for unrelated terms.
+(def ^:private test-mock-embeddings
+ {"belligerent" [1.0 0.0 0.0 0.0]
+ "combative" [0.99 0.01 0.0 0.0]
+ "bellicose" [0.0 1.0 0.0 0.0]
+ "quarrelsome" [0.01 0.99 0.0 0.0]
+ "quixotic" [0.0 0.0 1.0 0.0]
+ "ancillary" [0.0 0.0 0.0 0.9]
+ "adjunct" [0.0 0.0 0.0 1.0]
+ "baseline" [0.5 0.5 0.0 0.0]})
+
+(deftest split-keywords-only-test
+ (testing "search returns only exact matches for keyword terms when {:split-semantic-terms true}, regardless of whether semantic search is enabled\n"
+ (mt/with-test-user :rasta
+ (semantic.tu/with-test-db! {:mode :mock-initialized}
+ (with-and-without-semantic-search! test-mock-embeddings
+ (search.tu/with-new-search-and-legacy-search
+ (let [semantic-support? (search.engine/supported-engine? :search.engine/semantic)]
+ ;; "belligerent" and "bellicose" are semantically similar to our search terms
+ ;; ("combative", "quarrelsome") but should NOT match since we're only doing keyword search
+ (mt/with-temp [:model/Dashboard {id-1 :id} {:name "belligerent"}
+ :model/Dashboard {id-2 :id} {:name "bellicose"}
+ ;; "baseline" will match via keyword/fulltext search
+ :model/Dashboard {id-3 :id} {:name "baseline"}]
+ (when semantic-support?
+ (semantic.tu/index-all!))
+ (doseq [unified-disjunct-querying [false true]]
+ (testing (str "{unified-disjunct-querying " unified-disjunct-querying "}\n")
+ (let [base-query {:term-queries ["combative" "quarrelsome" "baseline"]
+ :semantic-queries []}
+ test-entity? (comp #{id-1 id-2 id-3} :id)
+ query (fn [unified-disjunct-querying]
+ (->> (search/search (assoc base-query
+ :experimental-opts
+ {:unified-disjunct-querying unified-disjunct-querying
+ :split-semantic-terms true}))
+ (filter test-entity?)
+ (map :name)))]
+ (testing "Semantic results are not returned for keyword terms"
+ (is (= #{"baseline"}
+ (set (query unified-disjunct-querying))))))))))))))))
+
+(deftest split-keyword-and-semantic-test
+ (testing "search returns only exact matches for keyword terms when {:split-semantic-terms true}\n"
+ (mt/with-test-user :rasta
+ (semantic.tu/with-test-db! {:mode :mock-initialized}
+ (with-semantic-search-if-available! test-mock-embeddings
+ (search.tu/with-new-search-and-legacy-search
+ ;; "belligerent" and "baseline" will match via keyword search (exact match in term-queries)
+ ;; "ancillary" and "adjunct" will match via semantic search (similar embeddings)
+ ;; "bellicose" and "quixotic" should NOT match (not in search terms)
+ (mt/with-temp [:model/Dashboard {id-1 :id} {:name "belligerent"}
+ :model/Dashboard {id-2 :id} {:name "bellicose"}
+ :model/Dashboard {id-3 :id} {:name "ancillary"}
+ :model/Dashboard {id-4 :id} {:name "adjunct"}
+ :model/Dashboard {id-5 :id} {:name "quixotic"}
+ :model/Dashboard {id-6 :id} {:name "baseline"}]
+ (semantic.tu/index-all!)
+ (doseq [unified-disjunct-querying [false true]]
+ (testing (str "{unified-disjunct-querying " unified-disjunct-querying "}\n")
+ (let [base-query {:term-queries ["baseline" "belligerent"]
+ :semantic-queries ["ancillary"]}
+ test-entity? (comp #{id-1 id-2 id-3 id-4 id-5 id-6} :id)
+ query (fn [unified-disjunct-querying]
+ (->> (search/search (assoc base-query
+ :experimental-opts
+ {:unified-disjunct-querying unified-disjunct-querying
+ :split-semantic-terms true}))
+ (filter test-entity?)
+ (map :name)))]
+ (testing "Semantic results are only returned for semantic terms"
+ (is (= #{"baseline" "belligerent" "ancillary" "adjunct"}
+ (set (query unified-disjunct-querying)))))))))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/test_util.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/test_util.clj
index 73fed24b551..1511be1ca88 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/test_util.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/test_util.clj
@@ -228,16 +228,44 @@
mock-documents-for-embeddings
base-embedding-vectors)))
+(def ^:dynamic *extra-mock-embeddings*
+ "Dynamic var for test-specific mock embeddings. Merged with [[mock-embeddings]] in [[get-mock-embedding]]."
+ nil)
+
+(defn- parse-entity-name
+ "Extract the entity name from embeddable text format.
+ E.g., '[dashboard]\\nname: My Dashboard\\n...' -> 'My Dashboard'"
+ [text]
+ (second (re-find #"(?m)^name:\s*(.+)$" text)))
+
(defn get-mock-embedding
- "Lookup the embedding for `text` in [[mock-embeddings]]."
+ "Lookup the embedding for `text` in [[*extra-mock-embeddings*]] first, then [[mock-embeddings]].
+ Lookups try both the raw text and the parsed entity name from embeddable text format."
[text]
- (get mock-embeddings text [0.01 0.02 0.03 0.04]))
+ (let [entity-name (parse-entity-name text)]
+ (or (get *extra-mock-embeddings* text)
+ (get *extra-mock-embeddings* entity-name)
+ (get mock-embeddings text)
+ (get mock-embeddings entity-name)
+ [0.01 0.02 0.03 0.04])))
(defn get-mock-embeddings-batch
"Lookup embeddings for multiple texts in [[mock-embeddings]]."
[texts]
(mapv get-mock-embedding texts))
+(defmacro with-mock-embeddings
+ "Bind extra mock embeddings for the duration of `body`. The embeddings map should be
+ a map from text strings to 4-dimensional vectors, e.g.:
+
+ (with-mock-embeddings
+ {\"belligerent\" [0.9 0.1 0.0 0.0]
+ \"combative\" [0.91 0.11 0.01 0.01]} ; similar vector = semantic match
+ ...)"
+ [embeddings-map & body]
+ `(binding [*extra-mock-embeddings* ~embeddings-map]
+ ~@body))
+
;;;; mock provider
(def mock-embedding-model
@@ -446,7 +474,7 @@
`(do-with-indexable-documents! (fn [] ~@body)))
(defn index-all!
- "Run indexer synchonously until we've exhausted polling all documents"
+ "Run indexer synchronously until we've exhausted polling all documents"
[]
(let [metadata-row {:indexer_last_poll Instant/EPOCH
:indexer_last_seen Instant/EPOCH}
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/api/search_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/api/search_test.clj
index e717aca535e..bfb4e450a56 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms/api/search_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms/api/search_test.clj
@@ -10,7 +10,7 @@
(use-fixtures :each (fn [thunk]
(binding [search.ingestion/*force-sync* true]
- (search.tu/with-new-search-if-available (thunk)))))
+ (search.tu/with-new-search-if-available-otherwise-legacy (thunk)))))
(deftest basic-search-test
(testing "Transform can be found with search"
diff --git a/src/metabase/search/appdb/core.clj b/src/metabase/search/appdb/core.clj
index 0bf603e6737..925b170fe09 100644
--- a/src/metabase/search/appdb/core.clj
+++ b/src/metabase/search/appdb/core.clj
@@ -1,5 +1,6 @@
(ns metabase.search.appdb.core
(:require
+ [clojure.string :as str]
[environ.core :as env]
[honey.sql.helpers :as sql.helpers]
[java-time.api :as t]
@@ -49,6 +50,10 @@
(#{"appdb" "semantic"} (some-> (search.settings/search-engine) name)))
(supported-db? (mdb/db-type))))
+(defmethod search.engine/disjunction :search.engine/appdb [_ terms]
+ (when (seq terms)
+ [(str/join " OR " (map #(str "(" % ")") terms))]))
+
(defn- parse-datetime [s]
(when s (OffsetDateTime/parse s)))
diff --git a/src/metabase/search/core.clj b/src/metabase/search/core.clj
index 4d7d5a96c94..1df3c034744 100644
--- a/src/metabase/search/core.clj
+++ b/src/metabase/search/core.clj
@@ -70,7 +70,7 @@
(defmethod analytics/initial-value :metabase-search/engine-default
[_ {:keys [engine]}]
- (if (= engine (name (search.impl/default-engine))) 1 0))
+ (if (= engine (name (search.engine/default-engine))) 1 0))
(defmethod analytics/initial-value :metabase-search/engine-active
[_ {:keys [engine]}]
diff --git a/src/metabase/search/engine.clj b/src/metabase/search/engine.clj
index 9d3437aef39..e42f4ab24a0 100644
--- a/src/metabase/search/engine.clj
+++ b/src/metabase/search/engine.clj
@@ -93,3 +93,20 @@
[engine]
(let [registered? #(contains? (methods supported-engine?) %)]
(some registered? (cons engine (ancestors engine)))))
+
+(defmulti disjunction
+ "Given multiple terms to search for, reduce this to a search expression that matches any of them in a single search.
+ If this is not possible, return a list of terms to be searched for separately."
+ {:arglists '([search-engine terms])}
+ (fn [search-engine _terms] search-engine))
+
+(defn default-engine
+ "In the absence of an explicit engine argument in a request, which engine should be used?"
+ []
+ ;; TODO (Chris 2025-11-07) It would be good to have a warning on start up whenever this is *not* what's configured.
+ (first (supported-engines)))
+
+(defmethod disjunction :default [_ terms] terms)
+
+(defmethod disjunction nil [_ terms]
+ (disjunction (default-engine) terms))
diff --git a/src/metabase/search/impl.clj b/src/metabase/search/impl.clj
index 843d4e264d0..256f5d0ffb4 100644
--- a/src/metabase/search/impl.clj
+++ b/src/metabase/search/impl.clj
@@ -215,12 +215,6 @@
(not (zero? v))
v))
-(defn default-engine
- "In the absence of an explicit engine argument in a request, which engine should be used?"
- []
- ;; TODO (Chris 2025-11-07) It would be good to have a warning on start up whenever this is *not* what's configured.
- (first (search.engine/supported-engines)))
-
(defn- parse-engine [value]
(or (when-not (str/blank? value)
(let [engine (keyword "search.engine" value)]
@@ -233,12 +227,12 @@
:else
engine)))
- (default-engine)))
+ (search.engine/default-engine)))
;; This forwarding is here for tests, we should clean those up.
(defn- apply-default-engine [{:keys [search-engine] :as search-ctx}]
- (let [default (default-engine)]
+ (let [default (search.engine/default-engine)]
(when (= default search-engine)
(throw (ex-info "Missing implementation for default search-engine" {:search-engine search-engine})))
(log/debugf "Missing implementation for %s so instead using %s" search-engine default)
diff --git a/src/metabase/search/in_place/legacy.clj b/src/metabase/search/in_place/legacy.clj
index 2b6e6d6c171..442a5dd6f3e 100644
--- a/src/metabase/search/in_place/legacy.clj
+++ b/src/metabase/search/in_place/legacy.clj
@@ -31,6 +31,11 @@
(defmethod search.engine/supported-engine? :search.engine/in-place [_]
true)
+(defmethod search.engine/disjunction :search.engine/in-place [_ terms]
+ ;; The default composition is disjunction with this engine.
+ (when (seq terms)
+ [(str/join " " terms)]))
+
(defn search-model->revision-model
"Return the appropriate revision model given a search model."
[model]
diff --git a/test/metabase/search/api_test.clj b/test/metabase/search/api_test.clj
index f5a8a92c999..66302f12b41 100644
--- a/test/metabase/search/api_test.clj
+++ b/test/metabase/search/api_test.clj
@@ -37,7 +37,7 @@
(def ^:private default-collection {:id false :name nil :authority_level nil :type nil})
(use-fixtures :each (fn [thunk] (binding [search.ingestion/*force-sync* true]
- (search.tu/with-new-search-if-available (thunk)))))
+ (search.tu/with-new-search-if-available-otherwise-legacy (thunk)))))
(def ^:private default-search-row
{:archived false
diff --git a/test/metabase/search/filter_test.clj b/test/metabase/search/filter_test.clj
index 6f15bdd5511..dc5487213f5 100644
--- a/test/metabase/search/filter_test.clj
+++ b/test/metabase/search/filter_test.clj
@@ -11,7 +11,7 @@
[metabase.search.test-util :as search.tu]))
(use-fixtures :each (fn [thunk] (binding [search.ingestion/*force-sync* true]
- (search.tu/with-new-search-if-available (thunk)))))
+ (search.tu/with-new-search-if-available-otherwise-legacy (thunk)))))
(defn- filter-keys []
(remove #{:ids} (map :context-key (vals search.config/filters))))
diff --git a/test/metabase/search/test_util.clj b/test/metabase/search/test_util.clj
index e1081a8a8f1..f1a6fafd0cb 100644
--- a/test/metabase/search/test_util.clj
+++ b/test/metabase/search/test_util.clj
@@ -31,22 +31,45 @@
~@body))))
#_{:clj-kondo/ignore [:metabase/test-helpers-use-non-thread-safe-functions]}
-(defmacro with-new-search-if-available
+(defmacro with-new-search-if-available*
+ "Create a temporary index table for the duration of the body."
+ [& body]
+ `(mt/with-dynamic-fn-redefs [search.engine/default-engine (constantly :search.engine/appdb)]
+ (with-temp-index-table
+ (search/reindex! {:async? false :in-place? true})
+ ~@body)))
+
+#_{:clj-kondo/ignore [:metabase/test-helpers-use-non-thread-safe-functions]}
+(defmacro with-new-search-if-available-otherwise-legacy
"Create a temporary index table for the duration of the body."
[& body]
`(if (search/supports-index?)
- (mt/with-dynamic-fn-redefs [search.impl/default-engine (constantly :search.engine/appdb)]
- (with-temp-index-table
- (search/reindex! {:async? false :in-place? true})
- ~@body))
+ (with-new-search-if-available* ~@body)
~@body))
+(defmacro with-new-search-if-available-without-fallback
+ "Create a temporary index table for the duration of the body.
+ Only runs if the appdb search engine is supported."
+ [& body]
+ `(when (search.engine/supported-engine? :search.engine/appdb)
+ (with-new-search-if-available* ~@body)))
+
(defmacro with-legacy-search
- "Ensure legacy search, which doesn't require an index, is used."
+ "Ensure legacy search, which doesn't require an index, is used.
+ When split-semantic-terms is enabled, semantic queries go to :search.engine/semantic
+ and keyword queries fall back to :search.engine/in-place."
[& body]
- `(mt/with-dynamic-fn-redefs [search.impl/default-engine (constantly :search.engine/in-place)]
+ `(mt/with-dynamic-fn-redefs [search.engine/default-engine (constantly :search.engine/in-place)
+ search.engine/supported-engines (constantly [:search.engine/semantic :search.engine/in-place])]
~@body))
+(defmacro with-new-search-and-legacy-search
+ "Run the body twice, once with the legacy search engine, and once with the appdb search engine."
+ [& body]
+ `(do
+ (with-legacy-search ~@body)
+ (with-new-search-if-available-without-fallback ~@body)))
+
(defmacro with-index-disabled
"Skip any index maintenance during this test."
[& body]