The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Search-model mapping clean up (#73032)
Chris Truter9afc3c63dc81
diff --git a/.gitignore b/.gitignoreindex 688fb9afc89..ad41d0335a5 100644--- a/.gitignore+++ b/.gitignore@@ -21,6 +21,7 @@/*.h2.db/*.lock.db/*.mv.db+/*.sql/*.trace.db/.babel_cache/.env*@@ -122,10 +123,8 @@ dev/serialization_deltas/mise.local.toml# lsp: ignore all but the config file-.lsp/*-!.lsp/config.edn-modules/drivers/*/.lsp/*-!modules/drivers/*/.lsp/config.edn+**/.lsp/*+!**/.lsp/config.edn**/.clj-kondo/.cache# clj-kondo: ignore all except our defined config@@ -178,3 +177,4 @@ CLAUDE.local.md__pycache__/.mcp.json.claude/worktrees+.claude/scheduled_tasks.lockdiff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/index.cljindex 93b0743a5ec..ab518847c40 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj@@ -609,12 +609,13 @@:order-by [[:keyword_rank :asc]]:limit (semantic-settings/semantic-search-results-limit)}))+(def ^:private ^:const max-cosine-distance "Cut-off used to filter semantic search results" 0.7)+(defn- semantic-search-query"Build a semantic search query using vector similarity with post-filtering to enable HNSW index usage."[index embedding search-context](let [filters (search-filters search-context)embedding-literal (format-embedding embedding)- max-cosine-distance 0.7;; Inner query: pure vector search to better trigger HNSW index vs. seqscan;; TODO: only pull in necessary extra columns from configured filtershnsw-query {:select (into common-search-columnsdiff --git a/src/metabase/metabot/core.clj b/src/metabase/metabot/core.cljindex e62f87a80c1..537b3774c0e 100644--- a/src/metabase/metabot/core.clj+++ b/src/metabase/metabot/core.clj@@ -3,6 +3,7 @@(:require[metabase.metabot.provider-util][metabase.metabot.scope]+ [metabase.metabot.search-models][metabase.metabot.usage][potemkin :as p]))@@ -15,7 +16,10 @@agent-query-constructagent-query-executeagent-search- agent-table-read])+ agent-table-read]+ [metabase.metabot.search-models+ entity-type->search-model+ search-model->entity-type])(p/import-vars[metabase.metabot.usagediff --git a/src/metabase/metabot/search_models.clj b/src/metabase/metabot/search_models.cljnew file mode 100644index 00000000000..9e2062e775d--- /dev/null+++ b/src/metabase/metabot/search_models.clj@@ -0,0 +1,24 @@+(ns metabase.metabot.search-models+ "Maps Metabot entity-type names to the `model` strings used by the search index."+ (:require+ [metabase.util :as u]))++(def ^:private entity->search+ "Entity-type strings whose Metabot name differs from the search `model` string. Unchanged names are omitted."+ {"model" "dataset"+ "question" "card"})++(def ^:private search->entity+ (u/for-map [[k v] entity->search] [v k]))++(defn entity-type->search-model+ "Metabot entity-type (string or keyword) → search `model` string.+ Unknown types yield themselves, even if they're not actual entity types."+ [entity-type]+ (let [s (if (keyword? entity-type) (name entity-type) (str entity-type))]+ (get entity->search s s)))++(defn search-model->entity-type+ "Inverse of [[entity-type->search-model]]."+ [search-model]+ (get search->entity search-model search-model))diff --git a/src/metabase/metabot/tools/search.clj b/src/metabase/metabot/tools/search.cljindex 55084b45d17..74b83b5d7f3 100644--- a/src/metabase/metabot/tools/search.clj+++ b/src/metabase/metabot/tools/search.clj@@ -1,12 +1,12 @@(ns metabase.metabot.tools.search"Search tool wrappers for Metabot v3."(:require- [clojure.set :as set][clojure.string :as str][medley.core :as m][metabase.api.common :as api][metabase.metabot.config :as metabot.config][metabase.metabot.scope :as scope]+ [metabase.metabot.search-models :as metabot.search-models][metabase.metabot.tmpl :as te][metabase.metabot.tools.shared :as shared][metabase.metabot.tools.shared.instructions :as instructions]@@ -25,19 +25,6 @@(def ^:private metabot-search-models#{"table" "dataset" "card" "dashboard" "metric" "database" "transform"})-(def ^:private search-model-mappings- "Maps metabot entity types to search engine model types"- {"model" "dataset"- "question" "card"})--(defn- entity-type->search-model- [entity-type]- (get search-model-mappings entity-type entity-type))--(defn- search-model->result-type- [search-model]- (get (set/map-invert search-model-mappings) search-model search-model))-(defn- postprocess-search-result"Transform a single search result to match the appropriate entity-specific schema."[{:keys [verified moderated_status collection] :as result}]@@ -45,7 +32,7 @@verified? (or (boolean verified) (= moderated_status "verified"))collection-info (select-keys collection [:id :name :authority_level])common-fields {:id (:id result)- :type (search-model->result-type model)+ :type (metabot.search-models/search-model->entity-type model):name (:name result):description (:description result):updated_at (:updated_at result)@@ -183,7 +170,7 @@:search-native-query search-native-query:weights weights})(let [search-models (if (seq entity-types)- (set (distinct (keep entity-type->search-model entity-types)))+ (set (distinct (keep metabot.search-models/entity-type->search-model entity-types)))metabot-search-models)_ (log/infof "[METABOT-SEARCH] Converted entity-types %s to search-models %s" entity-types search-models)metabot (t2/select-one :model/Metabot :entity_id (get-in metabot.config/metabot-config [metabot-id :entity-id] metabot-id))diff --git a/src/metabase/search/ingestion.clj b/src/metabase/search/ingestion.cljindex 6164d6afb75..ca0e6cbba5f 100644--- a/src/metabase/search/ingestion.clj+++ b/src/metabase/search/ingestion.clj@@ -196,14 +196,7 @@(comp (m/distinct-by (juxt :id :model)))))))(defn- search-items-reducible []- (let [models search.spec/search-models- ;; we're pushing indexed entities last in the search items reducible- ;; so that more important models gets indexed first, making the partial- ;; index more usable earlier- sorted-models (cond-> models- (contains? models "indexed-entity")- (-> (disj "indexed-entity") (concat ["indexed-entity"])))]- (reduce u/rconcat [] (map spec-index-reducible sorted-models))))+ (reduce u/rconcat [] (map spec-index-reducible search.spec/search-models)))(def ^:private max-document-error-logs 10)diff --git a/src/metabase/search/spec.clj b/src/metabase/search/spec.cljindex 5e28d001559..3db301ed612 100644--- a/src/metabase/search/spec.clj+++ b/src/metabase/search/spec.clj@@ -15,11 +15,25 @@[toucan2.tools.transformed :as t2.transformed]))(def search-models- "Set of search model string names. Sorted by order to index based on importance and amount of time to index"- (cond-> ["collection" "dashboard" "segment" "measure" "database" "action" "document" "transform"]- ;; metric/card/dataset moved to the end because they take a long time due to computing has_temporal_dim etc.- ;; table and indexed-entity moved to the end because there can be a large number of them- true (conj "table" "indexed-entity" "metric" "card" "dataset")))+ "Search model string names, ordered by indexing priority.+ Important / cheaper models come first so partial index is usable as soon as possible during a full index."+ ["collection"+ "dashboard"+ "segment"+ "measure"+ "database"+ "action"+ "document"+ "transform"+ ;; The following come last as they can be slow to index due to:+ ;; - cardinality (table, indexed-entity),+ ;; - cost (e.g. computing has_temporal_dim for cards)+ "table"+ "metric"+ "card"+ "dataset"+ ;; These can easily dwarf the cardinality of other entities, hence being dead last.+ "indexed-entity"])(def raw-spec-forms"Stores the raw (unevaluated) spec forms captured at macro expansion time.diff --git a/src/metabase/util.cljc b/src/metabase/util.cljcindex fb2f173d32b..26a0b52a24f 100644--- a/src/metabase/util.cljc+++ b/src/metabase/util.cljc@@ -1262,6 +1262,11 @@[reducible](reduce (fn [_ fst] (reduced fst)) nil reducible))+(defn rlast+ "Return last item from Reducible."+ [reducible]+ (reduce (fn [_ x] x) nil reducible))+(defn rconcat"Concatenate two Reducibles"[r1 r2]diff --git a/test/metabase/metabot/search_models_test.clj b/test/metabase/metabot/search_models_test.cljnew file mode 100644index 00000000000..0a940f95a00--- /dev/null+++ b/test/metabase/metabot/search_models_test.clj@@ -0,0 +1,18 @@+(ns metabase.metabot.search-models-test+ (:require+ [clojure.set :as set]+ [clojure.test :refer :all]+ [metabase.metabot.search-models :as sm]+ [metabase.search.spec :as search.spec]))++(deftest ^:parallel mappings-stay-in-sync-with-search-spec-test+ (let [mappings @#'sm/entity->search+ metabot-names (set (keys mappings))+ search-models (set search.spec/search-models)]+ (testing "translated values are valid search-model names"+ (is (set/subset? (set (vals mappings)) search-models)))+ (testing "metabot aliases don't collide with search-model names"+ (is (empty? (set/intersection metabot-names search-models))))+ (testing "every mapping round-trips — implies injectivity and a correct inverse"+ (doseq [k metabot-names]+ (is (= k (-> k sm/entity-type->search-model sm/search-model->entity-type)))))))diff --git a/test/metabase/search/appdb/index_test.clj b/test/metabase/search/appdb/index_test.cljindex b0ed38ed2c2..a74e5971041 100644--- a/test/metabase/search/appdb/index_test.clj+++ b/test/metabase/search/appdb/index_test.clj@@ -507,7 +507,7 @@:state "initial":creator_id (mt/user->id :rasta)}](model-index/add-values! model-index)- (is (= "dataset" (last (into [] (map :model) (search.ingestion/searchable-documents))))))))+ (is (= "indexed-entity" (:model (u/rlast (search.ingestion/searchable-documents))))))))(deftest ^:synchronized table-cleanup-test(when (search/supports-index?)