Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Search-model mapping clean up (#73032)

Chris Truter9afc3c63dc81
diff --git a/.gitignore b/.gitignore
index 688fb9afc89..ad41d0335a5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -21,6 +21,7 @@
/*.h2.db
/*.lock.db
/*.mv.db
+/*.sql
/*.trace.db
/.babel_cache
/.env*
@@ -122,10 +123,8 @@ dev/serialization_deltas/
mise.local.toml
# lsp: ignore all but the config file
-.lsp/*
-!.lsp/config.edn
-modules/drivers/*/.lsp/*
-!modules/drivers/*/.lsp/config.edn
+**/.lsp/*
+!**/.lsp/config.edn
**/.clj-kondo/.cache
# clj-kondo: ignore all except our defined config
@@ -178,3 +177,4 @@ CLAUDE.local.md
__pycache__/
.mcp.json
.claude/worktrees
+.claude/scheduled_tasks.lock
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj
index 93b0743a5ec..ab518847c40 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj
@@ -609,12 +609,13 @@
:order-by [[:keyword_rank :asc]]
:limit (semantic-settings/semantic-search-results-limit)}))
+(def ^:private ^:const max-cosine-distance "Cut-off used to filter semantic search results" 0.7)
+
(defn- semantic-search-query
"Build a semantic search query using vector similarity with post-filtering to enable HNSW index usage."
[index embedding search-context]
(let [filters (search-filters search-context)
embedding-literal (format-embedding embedding)
- max-cosine-distance 0.7
;; Inner query: pure vector search to better trigger HNSW index vs. seqscan
;; TODO: only pull in necessary extra columns from configured filters
hnsw-query {:select (into common-search-columns
diff --git a/src/metabase/metabot/core.clj b/src/metabase/metabot/core.clj
index e62f87a80c1..537b3774c0e 100644
--- a/src/metabase/metabot/core.clj
+++ b/src/metabase/metabot/core.clj
@@ -3,6 +3,7 @@
(:require
[metabase.metabot.provider-util]
[metabase.metabot.scope]
+ [metabase.metabot.search-models]
[metabase.metabot.usage]
[potemkin :as p]))
@@ -15,7 +16,10 @@
agent-query-construct
agent-query-execute
agent-search
- agent-table-read])
+ agent-table-read]
+ [metabase.metabot.search-models
+ entity-type->search-model
+ search-model->entity-type])
(p/import-vars
[metabase.metabot.usage
diff --git a/src/metabase/metabot/search_models.clj b/src/metabase/metabot/search_models.clj
new file mode 100644
index 00000000000..9e2062e775d
--- /dev/null
+++ b/src/metabase/metabot/search_models.clj
@@ -0,0 +1,24 @@
+(ns metabase.metabot.search-models
+ "Maps Metabot entity-type names to the `model` strings used by the search index."
+ (:require
+ [metabase.util :as u]))
+
+(def ^:private entity->search
+ "Entity-type strings whose Metabot name differs from the search `model` string. Unchanged names are omitted."
+ {"model" "dataset"
+ "question" "card"})
+
+(def ^:private search->entity
+ (u/for-map [[k v] entity->search] [v k]))
+
+(defn entity-type->search-model
+ "Metabot entity-type (string or keyword) → search `model` string.
+ Unknown types yield themselves, even if they're not actual entity types."
+ [entity-type]
+ (let [s (if (keyword? entity-type) (name entity-type) (str entity-type))]
+ (get entity->search s s)))
+
+(defn search-model->entity-type
+ "Inverse of [[entity-type->search-model]]."
+ [search-model]
+ (get search->entity search-model search-model))
diff --git a/src/metabase/metabot/tools/search.clj b/src/metabase/metabot/tools/search.clj
index 55084b45d17..74b83b5d7f3 100644
--- a/src/metabase/metabot/tools/search.clj
+++ b/src/metabase/metabot/tools/search.clj
@@ -1,12 +1,12 @@
(ns metabase.metabot.tools.search
"Search tool wrappers for Metabot v3."
(:require
- [clojure.set :as set]
[clojure.string :as str]
[medley.core :as m]
[metabase.api.common :as api]
[metabase.metabot.config :as metabot.config]
[metabase.metabot.scope :as scope]
+ [metabase.metabot.search-models :as metabot.search-models]
[metabase.metabot.tmpl :as te]
[metabase.metabot.tools.shared :as shared]
[metabase.metabot.tools.shared.instructions :as instructions]
@@ -25,19 +25,6 @@
(def ^:private metabot-search-models
#{"table" "dataset" "card" "dashboard" "metric" "database" "transform"})
-(def ^:private search-model-mappings
- "Maps metabot entity types to search engine model types"
- {"model" "dataset"
- "question" "card"})
-
-(defn- entity-type->search-model
- [entity-type]
- (get search-model-mappings entity-type entity-type))
-
-(defn- search-model->result-type
- [search-model]
- (get (set/map-invert search-model-mappings) search-model search-model))
-
(defn- postprocess-search-result
"Transform a single search result to match the appropriate entity-specific schema."
[{:keys [verified moderated_status collection] :as result}]
@@ -45,7 +32,7 @@
verified? (or (boolean verified) (= moderated_status "verified"))
collection-info (select-keys collection [:id :name :authority_level])
common-fields {:id (:id result)
- :type (search-model->result-type model)
+ :type (metabot.search-models/search-model->entity-type model)
:name (:name result)
:description (:description result)
:updated_at (:updated_at result)
@@ -183,7 +170,7 @@
:search-native-query search-native-query
:weights weights})
(let [search-models (if (seq entity-types)
- (set (distinct (keep entity-type->search-model entity-types)))
+ (set (distinct (keep metabot.search-models/entity-type->search-model entity-types)))
metabot-search-models)
_ (log/infof "[METABOT-SEARCH] Converted entity-types %s to search-models %s" entity-types search-models)
metabot (t2/select-one :model/Metabot :entity_id (get-in metabot.config/metabot-config [metabot-id :entity-id] metabot-id))
diff --git a/src/metabase/search/ingestion.clj b/src/metabase/search/ingestion.clj
index 6164d6afb75..ca0e6cbba5f 100644
--- a/src/metabase/search/ingestion.clj
+++ b/src/metabase/search/ingestion.clj
@@ -196,14 +196,7 @@
(comp (m/distinct-by (juxt :id :model)))))))
(defn- search-items-reducible []
- (let [models search.spec/search-models
- ;; we're pushing indexed entities last in the search items reducible
- ;; so that more important models gets indexed first, making the partial
- ;; index more usable earlier
- sorted-models (cond-> models
- (contains? models "indexed-entity")
- (-> (disj "indexed-entity") (concat ["indexed-entity"])))]
- (reduce u/rconcat [] (map spec-index-reducible sorted-models))))
+ (reduce u/rconcat [] (map spec-index-reducible search.spec/search-models)))
(def ^:private max-document-error-logs 10)
diff --git a/src/metabase/search/spec.clj b/src/metabase/search/spec.clj
index 5e28d001559..3db301ed612 100644
--- a/src/metabase/search/spec.clj
+++ b/src/metabase/search/spec.clj
@@ -15,11 +15,25 @@
[toucan2.tools.transformed :as t2.transformed]))
(def search-models
- "Set of search model string names. Sorted by order to index based on importance and amount of time to index"
- (cond-> ["collection" "dashboard" "segment" "measure" "database" "action" "document" "transform"]
- ;; metric/card/dataset moved to the end because they take a long time due to computing has_temporal_dim etc.
- ;; table and indexed-entity moved to the end because there can be a large number of them
- true (conj "table" "indexed-entity" "metric" "card" "dataset")))
+ "Search model string names, ordered by indexing priority.
+ Important / cheaper models come first so partial index is usable as soon as possible during a full index."
+ ["collection"
+ "dashboard"
+ "segment"
+ "measure"
+ "database"
+ "action"
+ "document"
+ "transform"
+ ;; The following come last as they can be slow to index due to:
+ ;; - cardinality (table, indexed-entity),
+ ;; - cost (e.g. computing has_temporal_dim for cards)
+ "table"
+ "metric"
+ "card"
+ "dataset"
+ ;; These can easily dwarf the cardinality of other entities, hence being dead last.
+ "indexed-entity"])
(def raw-spec-forms
"Stores the raw (unevaluated) spec forms captured at macro expansion time.
diff --git a/src/metabase/util.cljc b/src/metabase/util.cljc
index fb2f173d32b..26a0b52a24f 100644
--- a/src/metabase/util.cljc
+++ b/src/metabase/util.cljc
@@ -1262,6 +1262,11 @@
[reducible]
(reduce (fn [_ fst] (reduced fst)) nil reducible))
+(defn rlast
+ "Return last item from Reducible."
+ [reducible]
+ (reduce (fn [_ x] x) nil reducible))
+
(defn rconcat
"Concatenate two Reducibles"
[r1 r2]
diff --git a/test/metabase/metabot/search_models_test.clj b/test/metabase/metabot/search_models_test.clj
new file mode 100644
index 00000000000..0a940f95a00
--- /dev/null
+++ b/test/metabase/metabot/search_models_test.clj
@@ -0,0 +1,18 @@
+(ns metabase.metabot.search-models-test
+ (:require
+ [clojure.set :as set]
+ [clojure.test :refer :all]
+ [metabase.metabot.search-models :as sm]
+ [metabase.search.spec :as search.spec]))
+
+(deftest ^:parallel mappings-stay-in-sync-with-search-spec-test
+ (let [mappings @#'sm/entity->search
+ metabot-names (set (keys mappings))
+ search-models (set search.spec/search-models)]
+ (testing "translated values are valid search-model names"
+ (is (set/subset? (set (vals mappings)) search-models)))
+ (testing "metabot aliases don't collide with search-model names"
+ (is (empty? (set/intersection metabot-names search-models))))
+ (testing "every mapping round-trips — implies injectivity and a correct inverse"
+ (doseq [k metabot-names]
+ (is (= k (-> k sm/entity-type->search-model sm/search-model->entity-type)))))))
diff --git a/test/metabase/search/appdb/index_test.clj b/test/metabase/search/appdb/index_test.clj
index b0ed38ed2c2..a74e5971041 100644
--- a/test/metabase/search/appdb/index_test.clj
+++ b/test/metabase/search/appdb/index_test.clj
@@ -507,7 +507,7 @@
:state "initial"
:creator_id (mt/user->id :rasta)}]
(model-index/add-values! model-index)
- (is (= "dataset" (last (into [] (map :model) (search.ingestion/searchable-documents))))))))
+ (is (= "indexed-entity" (:model (u/rlast (search.ingestion/searchable-documents))))))))
(deftest ^:synchronized table-cleanup-test
(when (search/supports-index?)