The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
search: add :metabot weights context and curation-tier scorers (#74244)
Mihael Konjević57db5261791b
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration/impl.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration/impl.cljindex 3b17db7f9fb..c513c26b7c0 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration/impl.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration/impl.clj@@ -37,29 +37,59 @@(def dynamic-schema-version"Code version of dynamic schema (index_table_xyzs). If higher than what's found in db dynamic schema migration willbe attempted."- 2)+ 3)-(defn- add-personal-owner-id-column!- "Migration 2: Add `personal_owner_id` column to index tables for SQL-level personal collection filtering."- [tx index-metadata]- (let [index-metadata (keyword (:metadata-table-name index-metadata))- execute! (fn [q] (jdbc/execute! tx (sql/format q)))- table-names (->> (execute! {:select-distinct [:table_name]- :from [index-metadata]- :where [[:< :index_version 2]]})- (mapcat vals))]+(defn- alter-index-tables!+ "Run `alter-fn` against each existing index table whose `index_version` is below `target-version`, then bump those+ rows' `index_version` to `target-version`. `alter-fn` is `(fn [execute! table-name])`. Tables listed in+ `index_metadata` but missing from the database (e.g. dropped externally) are skipped — leaving the metadata row+ in place to be cleaned up or re-created elsewhere."+ [tx index-metadata target-version alter-fn]+ (let [metadata-table (keyword (:metadata-table-name index-metadata))+ execute! (fn [q] (jdbc/execute! tx (sql/format q)))+ candidate-names (->> (execute! {:select-distinct [:table_name]+ :from [metadata-table]+ :where [[:< :index_version target-version]]})+ (mapcat vals))+ existing-names (when (seq candidate-names)+ (->> (execute! {:select [:tablename]+ :from [:pg_tables]+ :where [:in :tablename (vec candidate-names)]})+ (mapcat vals)+ set))+ table-names (filter existing-names candidate-names)](when (seq table-names)(doseq [table-name table-names]- (execute! {:alter-table [(keyword table-name)]- :add-column [[:personal_owner_id :int :if-not-exists]]}))- (execute! {:update index-metadata- :set {:index_version 2}+ (alter-fn execute! table-name))+ (execute! {:update metadata-table+ :set {:index_version target-version}:where [[:in :table_name (vec table-names)]]}))))+(defn- add-personal-owner-id-column!+ "Migration 2: Add `personal_owner_id` column to index tables for SQL-level personal collection filtering."+ [tx index-metadata]+ (alter-index-tables! tx index-metadata 2+ (fn [execute! table-name]+ (execute! {:alter-table [(keyword table-name)]+ :add-column [[:personal_owner_id :int :if-not-exists]]}))))++(defn- add-collection-type-and-data-layer-columns!+ "Migration 3: Add `collection_type` and `data_layer` columns to index tables for the new ranking signals+ (`:library` on `collection_type`; `:data-layer` per-tier weights on `data_layer`)."+ [tx index-metadata]+ (alter-index-tables! tx index-metadata 3+ (fn [execute! table-name]+ (execute! {:alter-table [(keyword table-name)]+ :add-column [[:collection_type :text :if-not-exists]]})+ (execute! {:alter-table [(keyword table-name)]+ :add-column [[:data_layer :text :if-not-exists]]}))))+(defn migrate-dynamic-schema!"Migrate runtime-managed schema, ie. schema of `index_table_...` tables. Migration author is responsible for removingleftovers if necessary."[tx {:keys [index-metadata] :as _opts}];; migration 1: all tables dropped in schema migration in single function call;; migration 2: add personal_owner_id column to index tables- (add-personal-owner-id-column! tx index-metadata))+ ;; migration 3: add collection_type and data_layer columns to index tables+ (add-personal-owner-id-column! tx index-metadata)+ (add-collection-type-and-data-layer-columns! tx index-metadata))diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/index.cljindex b0e001ed766..f9c517f3214 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj@@ -66,6 +66,8 @@[:official_collection :boolean][:pinned :boolean][:verified :boolean]+ [:collection_type :text]+ [:data_layer :text][:dashboardcard_count :int][:view_count :int][:created_at :timestamp-with-time-zone [:default [:raw "CURRENT_TIMESTAMP"]] :not-null]@@ -150,7 +152,7 @@"Convert a document to a database record with a provided embedding."[owner-ids {:keys [model id embedding searchable_text embeddable_text native_query created_at creator_id updated_atlast_editor_id archived verified official_collection database_id collection_id display_type legacy_input- pinned dashboardcard_count view_count last_viewed_at] :as doc}]+ pinned dashboardcard_count view_count last_viewed_at collection_type data_layer] :as doc}]{:model model:model_id id:collection_id collection_id@@ -165,6 +167,8 @@:official_collection (some-> official_collection to-boolean):pinned (some-> pinned to-boolean):verified (some-> verified to-boolean)+ :collection_type collection_type+ :data_layer data_layer:dashboardcard_count dashboardcard_count:view_count view_count:model_created_at (some-> created_at to-instant)@@ -280,7 +284,7 @@(let [table-name (or table-name (model-table-name embedding-model))]{:embedding-model embedding-model:table-name table-name- :version 2}))+ :version 3}))(defn- upsert-embedding!-fn [connectable index text->docs](fn [text->embedding]@@ -580,6 +584,8 @@[:official_collection :official_collection][:pinned :pinned][:verified :verified]+ [:collection_type :collection_type]+ [:data_layer :data_layer][:dashboardcard_count :dashboardcard_count][:view_count :view_count][:model_created_at :model_created_at]diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/migration_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/migration_test.cljindex e7d98456725..265625c47ec 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/migration_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/migration_test.clj@@ -173,11 +173,13 @@:index_metadata/table_name)](is (=? #{"archived""collection_id"+ "collection_type""content""created_at""creator_id""dashboardcard_count""database_id"+ "data_layer""display_type""embedding""id"diff --git a/src/metabase/collections/models/collection.clj b/src/metabase/collections/models/collection.cljindex 9d3dc597788..5ed09fca473 100644--- a/src/metabase/collections/models/collection.clj+++ b/src/metabase/collections/models/collection.clj@@ -73,6 +73,13 @@"The value of the `:type` field for collections that only allow metrics.""library-metrics")+(def library-collection-types+ "All library `:type` values — collections users curate as the canonical place to find content.+ Kept as a set so callers (e.g. search ranking) can enumerate them without hard-coding strings."+ #{library-collection-type+ library-data-collection-type+ library-metrics-collection-type})+(def ^:constant tenant-specific-root-collection-type"The value of the `:type` field for root collections that belong to a single tenant""tenant-specific-root-collection")@@ -182,6 +189,47 @@[](t2/select-one :model/Collection :type library-collection-type))+(def ^{:arglists '([id])} root-collection-type-by-id+ "Return the `:type` of the top-level (root) collection with the given `id`, or `nil` if no+ top-level collection has that id. Caching is keyed by `[app-db-id collection-id]` per the+ `metabase.app-db.core/memoize-for-application-db` docstring — different ids never share cache+ entries, which keeps tests isolated even when other tests run ingestion in parallel.++ Top-level collections change infrequently, so a 1-hour TTL is fine."+ (memoize/ttl+ ^{::memoize/args-fn (fn [[id]] [(mdb/unique-identifier) id])}+ (fn [id]+ (when id+ (t2/select-one-fn :type :model/Collection :id id :location "/")))+ :ttl/threshold (* 60 60 1000)))++(defn root-collection-type+ "Return the `:type` of the top-level ancestor collection given a collection's materialized path and+ its own type/id. Used by the search ranking spec to power the `:library` scorer for items nested+ inside library trees.++ Resolution rules:+ - `:collection_id` `nil` → `nil` (item has no collection)+ - location is `\"/\"` or `nil` → the collection's own `:collection_type` (it IS a root)+ - otherwise → parse the first id out of the materialized path and look up its type++ Reads `:collection_id`, `:collection_location`, `:collection_type` from the ingestion record map."+ [{:keys [collection_id collection_location collection_type]}]+ (cond+ (nil? collection_id)+ nil++ (or (nil? collection_location) (= "/" collection_location))+ collection_type++ :else+ (when-let [root-id (some-> collection_location+ (str/split #"/")+ (->> (remove str/blank?))+ first+ parse-long)]+ (root-collection-type-by-id root-id))))+(def library-entity-id"The entity_id for the Library collection.""librarylibrarylibrary")@@ -2405,20 +2453,25 @@(search.spec/define-spec "collection"{:model :model/Collection- :attrs {:collection-id :id- :creator-id false- :database-id false- :archived true- :created-at true+ :attrs {:collection-id :id+ :collection-type :type+ :collection-location :location+ :root-collection-type {:fn root-collection-type}+ :creator-id false+ :database-id false+ :archived true+ :created-at true;; intentionally not tracked- :updated-at false}+ :updated-at false}:search-terms [:name]:render-terms {:archived-directly true;; Why not make this a search term? I suspect it was just overlooked before.:description true:collection_authority_level :authority_level:collection_name :name- :collection_type :type+ ;; `:location` is read by the toucan2 effective-location hydration when collection+ ;; results pass through `metabase.search.impl/add-collection-effective-location`.+ ;; Keep the snake_case `location` key flowing alongside the indexed `collection_location`.:location true}:where [:or [:= :namespace nil][:= :namespace "analytics"]diff --git a/src/metabase/dashboards/models/dashboard.clj b/src/metabase/dashboards/models/dashboard.cljindex 0d5d7cf664f..00144306b9a 100644--- a/src/metabase/dashboards/models/dashboard.clj+++ b/src/metabase/dashboards/models/dashboard.clj@@ -482,14 +482,16 @@:verified [:= "verified" :mr.status]:view-count true:created-at true- :updated-at true}+ :updated-at true+ :collection-type :collection.type+ :collection-location :collection.location+ :root-collection-type {:fn collection/root-collection-type}}:search-terms [:name :description]:render-terms {:archived-directly true:collection-authority_level :collection.authority_level:collection-name :collection.name;; This is used for legacy ranking, in future it will be replaced by :pinned:collection-position true- :collection-type :collection.type:moderated-status :mr.status}:where []:bookmark [:model/DashboardBookmark [:anddiff --git a/src/metabase/queries/models/card.clj b/src/metabase/queries/models/card.cljindex fde60d21b8f..7be1ab9a3bd 100644--- a/src/metabase/queries/models/card.clj+++ b/src/metabase/queries/models/card.clj@@ -1463,17 +1463,18 @@:created-at true:updated-at true:display-type :this.display+ :collection-type :collection.type+ :collection-location :collection.location+ :root-collection-type {:fn collection/root-collection-type}:temporal-info {:fn extract-temporal-info:fields [:dataset_query :query_type]:provides [:has-temporal-dim :non-temporal-dim-ids]}}:search-terms [:name :description]:render-terms {:archived-directly true:collection-authority_level :collection.authority_level- :collection-location :collection.location:collection-name :collection.name;; This is used for legacy ranking, in future it will be replaced by :pinned:collection-position true- :collection-type :collection.type;; This field can become stale, unless we change to calculate it just-in-time.:display true:moderated-status :mr.status}diff --git a/src/metabase/search/appdb/scoring.clj b/src/metabase/search/appdb/scoring.cljindex 2fbfa86065c..e053e956575 100644--- a/src/metabase/search/appdb/scoring.clj+++ b/src/metabase/search/appdb/scoring.clj@@ -2,6 +2,7 @@(:require[clojure.core.memoize :as memoize][honey.sql.helpers :as sql.helpers]+ [metabase.collections.models.collection :as collection][metabase.config.core :as config][metabase.premium-features.core :refer [defenterprise]][metabase.search.appdb.index :as search.index]@@ -62,7 +63,27 @@:prefix (if search-string;; in this case, we need to transform the string into a pattern in code, so forced to use helper(search.scoring/prefix [:lower :search_index.name] (u/lower-case-en search-string))- [:inline 0])}))+ [:inline 0])+ ;; :library matches items whose root (top-level) ancestor collection is one of the library types.+ ;; The :root-collection-type attr is computed at ingestion time by walking the collection's+ ;; materialized path (see metabase.collections.models.collection/root-collection-type), so items+ ;; in arbitrarily deep sub-collections of a library tree still match here.+ :library [:case+ (into [:or]+ (for [t (sort collection/library-collection-types)]+ [:= :search_index.root_collection_type [:inline t]]))+ [:inline 1]+ :else [:inline 0]]+ ;; :data-layer mirrors the :model/* pattern — one scorer, per-tier weights live under :data-layer/*+ ;; (see metabase.search.config/scorer-param). Final/internal/hidden are mutually exclusive.+ :data-layer [:case+ [:= :search_index.data_layer [:inline "final"]]+ [:inline (or (search.config/scorer-param search-ctx :data-layer :final) 0)]+ [:= :search_index.data_layer [:inline "internal"]]+ [:inline (or (search.config/scorer-param search-ctx :data-layer :internal) 0)]+ [:= :search_index.data_layer [:inline "hidden"]]+ [:inline (or (search.config/scorer-param search-ctx :data-layer :hidden) 0)]+ :else [:inline 0]]}))(defenterprise scorers"Return the select-item expressions used to calculate the score for each search result."diff --git a/src/metabase/search/config.clj b/src/metabase/search/config.cljindex 6967a7382cb..ac3ed950247 100644--- a/src/metabase/search/config.clj+++ b/src/metabase/search/config.clj@@ -107,7 +107,15 @@{:model/table 1:model/dataset 1:model/metric 1- :model/question 0}})+ :model/question 0}+ :metabot+ {:library 100+ :official-collection 80+ :verified 80+ :data-layer 1 ; overall multiplier; per-tier weights live under :data-layer/* below+ :data-layer/final 33+ :data-layer/internal 10+ :data-layer/hidden 1}})(def ^:private FilterDef"A relaxed definition, capturing how we can write the filter - with some fields omitted."diff --git a/src/metabase/search/ingestion.clj b/src/metabase/search/ingestion.cljindex 78e829bb03a..c27fab7e64c 100644--- a/src/metabase/search/ingestion.clj+++ b/src/metabase/search/ingestion.clj@@ -140,7 +140,7 @@(update :archived boolean)(assoc:display_data (display-data m)- :legacy_input (json/encode (dissoc m :pinned :view_count :last_viewed_at :native_query :dataset_query))+ :legacy_input (json/encode (apply dissoc m search.spec/legacy-input-excluded-keys)):searchable_text (searchable-text m):embeddable_text (embeddable-text m)))](merge fn-results sql-results)))diff --git a/src/metabase/search/spec.clj b/src/metabase/search/spec.cljindex 3db301ed612..c77f9edca46 100644--- a/src/metabase/search/spec.clj+++ b/src/metabase/search/spec.clj@@ -83,6 +83,17 @@(mapcat :fields)distinct))+(def legacy-input-excluded-keys+ "Keys present on the ingestion document `m` that must NOT be encoded into `legacy_input`. These are+ internal signals (ranking, filtering, ingestion bookkeeping) that the search API response should not+ surface to clients. Consumers: `metabase.search.ingestion/->document`."+ ;; `:collection_type` and `:collection_location` deliberately stay IN `legacy_input`:+ ;; - `metabase.search.impl/serialize` reads `:collection_type` to build the response's `:collection.type`+ ;; - `metabase.search.impl/add-dataset-collection-hierarchy` reads `:collection_location` to hydrate+ ;; `:collection_effective_ancestors`, and the collection-result hydration path reads `:location` from+ ;; the toucan instance (which the render-term keeps populated).+ #{:pinned :view_count :last_viewed_at :native_query :dataset_query :data_layer})+(def attr-types"The abstract types of each attribute."{:archived :boolean@@ -108,7 +119,11 @@:temporal-info nil:display-type :text:is-published :boolean- :source-type :text})+ :source-type :text+ :collection-type :text+ :collection-location :text+ :root-collection-type :text+ :data-layer :text})(def ^:private explicit-attrs"These attributes must be explicitly defined, omitting them could be a source of bugs."@@ -132,7 +147,11 @@:updated-at:temporal-info:is-published- :source-type])+ :source-type+ :collection-type ;; surfaced for downstream consumers (metabase.search.impl/serialize)+ :collection-location ;; surfaced for downstream consumers (add-dataset-collection-hierarchy)+ :root-collection-type ;; indexed for :library scorer — type of the top-level ancestor collection+ :data-layer]) ;; indexed for the :data-layer scorer (table.data_layer; per-tier weights under :data-layer/*)distinctvec))diff --git a/src/metabase/warehouse_schema/models/table.clj b/src/metabase/warehouse_schema/models/table.cljindex 121046d1510..49e02fd08df 100644--- a/src/metabase/warehouse_schema/models/table.clj+++ b/src/metabase/warehouse_schema/models/table.clj@@ -658,15 +658,19 @@{:model :model/Table:attrs {;; legacy search uses :active for this, but then has a rule to only ever show active tables;; so we moved that to the where clause- :archived false+ :archived false;; For published tables with no collection, we want to show "root" as the collection id- :collection-id true- :creator-id false- :database-id :db_id- :view-count true- :created-at true- :updated-at true- :is-published :is_published}+ :collection-id true+ :creator-id false+ :database-id :db_id+ :view-count true+ :created-at true+ :updated-at true+ :is-published :is_published+ :collection-type :collection.type+ :collection-location :collection.location+ :root-collection-type {:fn collection/root-collection-type}+ :data-layer :data_layer}:search-terms {:name search.spec/explode-camel-case:display_name true:description true}@@ -677,14 +681,12 @@:table-schema :schema:database-name :db.name:collection-authority_level :collection.authority_level- :collection-location :collection.location;; For published tables with no collection, show "Our analytics" as the collection name:collection-name [:coalesce :collection.name[:case[:and :this.is_published[:= :this.collection_id nil]] [:inline "Our analytics"]- :else nil]]- :collection-type :collection.type}+ :else nil]]}:where [:and:active[:= :visibility_type nil]diff --git a/test/metabase/search/appdb/scoring_test.clj b/test/metabase/search/appdb/scoring_test.cljindex b8a53b9f2f5..483e405b7a5 100644--- a/test/metabase/search/appdb/scoring_test.clj+++ b/test/metabase/search/appdb/scoring_test.clj@@ -316,3 +316,62 @@(is (= [["card" 2 "this card is aerie mon"]["card" 1 "crow's fly card"]](search-results :mine "card" {:current-user-id rasta}))))))++(deftest library-test+ (testing "Library-collection cards rank above non-library cards"+ ;; Real Collections are needed in appdb so the `:root-collection-type` fn attr can resolve the+ ;; top-level ancestor's `:type` from each row's `:collection_location` materialized path.+ (mt/with-temp [:model/Collection lib {:name "lib top" :type "library" :location "/"}+ :model/Collection lib-data {:name "lib data" :type "library-data" :location "/"}+ :model/Collection lib-met {:name "lib metrics" :type "library-metrics" :location "/"}+ :model/Collection sub {:name "sub of lib" :location (format "/%d/" (:id lib))}+ :model/Collection sub-sub {:name "sub of sub" :location (format "/%d/%d/" (:id lib) (:id sub))}+ :model/Collection other {:name "non-library" :location "/"}]+ (with-index-contents+ [{:model "card" :id 1 :name "card plain" :collection_id (:id other) :collection_location (:location other)}+ {:model "card" :id 2 :name "card in library" :collection_id (:id lib) :collection_location (:location lib) :collection_type "library"}+ {:model "card" :id 3 :name "card in library-data" :collection_id (:id lib-data) :collection_location (:location lib-data) :collection_type "library-data"}+ {:model "card" :id 4 :name "card in lib-metrics" :collection_id (:id lib-met) :collection_location (:location lib-met) :collection_type "library-metrics"}+ {:model "card" :id 5 :name "card in sub of lib" :collection_id (:id sub) :collection_location (:location sub)}+ {:model "card" :id 6 :name "card in sub of sub" :collection_id (:id sub-sub) :collection_location (:location sub-sub)}+ {:model "card" :id 7 :name "card in trash" :collection_type "trash"}]+ (let [library-id? #{2 3 4 5 6}+ in-library? (fn [[_ id _]] (boolean (library-id? id)))]+ (testing "with positive :library weight, items inside library trees come first"+ (is (= [true true true true true false false]+ (map in-library? (with-weights {:library 1} (search-results* "card"))))))+ (testing "with negative :library weight, library items come last"+ (is (= [false false true true true true true]+ (map in-library? (with-weights {:library -1} (search-results* "card")))))))))))++(deftest ^:parallel data-layer-test+ (testing ":data-layer scorer reads the active per-tier weight via :data-layer/* params"+ (with-index-contents+ [{:model "table" :id 1 :name "table no layer"}+ {:model "table" :id 2 :name "table final" :data_layer "final"}+ {:model "table" :id 3 :name "table internal" :data_layer "internal"}+ {:model "table" :id 4 :name "table hidden" :data_layer "hidden"}]+ (testing "final tables lead when only :data-layer/final is positive"+ (is (= 2 (-> (with-weights {:data-layer 1 :data-layer/final 1}+ (search-results* "table"))+ first second))))+ (testing "internal tables lead when only :data-layer/internal is positive"+ (is (= 3 (-> (with-weights {:data-layer 1 :data-layer/internal 1}+ (search-results* "table"))+ first second))))+ (testing "hidden tables lead when only :data-layer/hidden is positive"+ (is (= 4 (-> (with-weights {:data-layer 1 :data-layer/hidden 1}+ (search-results* "table"))+ first second))))))+ (testing "Metabot tier ordering: final > internal > hidden when all weights active"+ (with-index-contents+ [{:model "table" :id 1 :name "metabot table final" :data_layer "final"}+ {:model "table" :id 2 :name "metabot table internal" :data_layer "internal"}+ {:model "table" :id 3 :name "metabot table hidden" :data_layer "hidden"}]+ (is (= [1 2 3]+ (->> (with-weights {:data-layer 1+ :data-layer/final 33+ :data-layer/internal 10+ :data-layer/hidden 1}+ (search-results* "metabot table"))+ (map second)))))))diff --git a/test/metabase/search/config_test.clj b/test/metabase/search/config_test.cljindex a258ee06bba..c2ce587c9d3 100644--- a/test/metabase/search/config_test.clj+++ b/test/metabase/search/config_test.clj@@ -22,3 +22,17 @@(search.config/filter-default :search.engine/in-place :command-palette :filter-items-in-personal-collection)))(is (= "exclude-others"(search.config/filter-default :search.engine/in-place :search-app :filter-items-in-personal-collection)))))++(deftest metabot-weights-test+ (testing "the :metabot context defines the tuned curation tier weights"+ (is (=? {:library 100+ :official-collection 80+ :verified 80+ :data-layer 1+ :data-layer/final 33+ :data-layer/internal 10+ :data-layer/hidden 1}+ (search.config/weights {:context :metabot}))))+ (testing "request-level :weights override beats :metabot static weights"+ (is (= 7 (-> (search.config/weights {:context :metabot :weights {:library 7}})+ :library)))))diff --git a/test/metabase/search/spec_test.clj b/test/metabase/search/spec_test.cljindex c1cc41569c3..dbef644ae48 100644--- a/test/metabase/search/spec_test.clj+++ b/test/metabase/search/spec_test.clj@@ -101,7 +101,8 @@{:search-model "table",:fields#{:active :description :schema :name :id :db_id :initial_sync_status :display_name- :visibility_type :view_count :created_at :updated_at :collection_id :is_published}+ :visibility_type :view_count :created_at :updated_at :collection_id :is_published+ :data_layer}:where [:= :updated.id :this.id]}},:Database #{{:search-model "table":fields #{:name :router_database_id}