The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Boost library-collection items in search ranking (#74806)
Chris Truterd587fb7ffee1
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration/impl.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration/impl.cljindex c513c26b7c0..a1bdbe7b85b 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration/impl.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration/impl.clj@@ -2,7 +2,10 @@(:require[honey.sql :as sql][metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata]- [next.jdbc :as jdbc]))+ [metabase.util :as u]+ [metabase.util.log :as log]+ [next.jdbc :as jdbc]+ [toucan2.core :as t2]))(def schema-version"Version to compare the [[metabase-enterprise.semantic-search.db.migration/db-version]] with. If this is higher,@@ -37,7 +40,7 @@(def dynamic-schema-version"Code version of dynamic schema (index_table_xyzs). If higher than what's found in db dynamic schema migration willbe attempted."- 3)+ 4)(defn- alter-index-tables!"Run `alter-fn` against each existing index table whose `index_version` is below `target-version`, then bump those@@ -74,8 +77,10 @@:add-column [[:personal_owner_id :int :if-not-exists]]}))))(defn- add-collection-type-and-data-layer-columns!- "Migration 3: Add `collection_type` and `data_layer` columns to index tables for the new ranking signals- (`:library` on `collection_type`; `:data-layer` per-tier weights on `data_layer`)."+ "Migration 3: Add `collection_type` and `data_layer` columns to index tables.+ `collection_type` is surfaced for downstream consumers (e.g. `metabase.search.impl/serialize`);+ `data_layer` powers the `:data-layer` scorer (per-tier weights under `:data-layer/*`).+ The `:library` scorer was reworked in migration 4 to read `root_collection_type` instead."[tx index-metadata](alter-index-tables! tx index-metadata 3(fn [execute! table-name]@@ -84,6 +89,60 @@(execute! {:alter-table [(keyword table-name)]:add-column [[:data_layer :text :if-not-exists]]}))))+(defn- library-root-type-by-collection-id+ "Map `collection-id → root-collection-type` for every collection in a Library tree.+ Returns `{}` and logs a warning if the appdb lookup fails."+ [library-types]+ ;; Catch covers test setups that exercise pgvector before the appdb schema is up;+ ;; production semantic-search init always runs after appdb migration.+ (try+ (u/for-map [{root-id :id root-type :type} (t2/select [:model/Collection :id :type]+ :type [:in library-types]+ :location "/")+ coll-id (cons root-id (t2/select-pks-set :model/Collection+ :location [:like (str "/" root-id "/%")]))]+ [coll-id root-type])+ (catch Exception e+ (log/warn e "Skipping Library forest backfill — appdb lookup failed")+ {})))++(defn- add-root-collection-type-column!+ "Migration 4: add `root_collection_type` to index tables, backfilling first from each row's+ gate document and then from an appdb sweep of the Library forest. Drives the `:library` scorer."+ [tx index-metadata]+ ;; Library types are hardcoded: migrations are frozen snapshots, and semantic-search doesn't `:use` `collections`.+ (let [gate-table (:gate-table-name index-metadata)+ library-types ["library" "library-data" "library-metrics"]+ ;; Resolved once — the same map applies to every index table.+ root-type-by-coll-id (library-root-type-by-collection-id library-types)]+ (alter-index-tables!+ tx index-metadata 4+ (fn [execute! table-name]+ (let [kw-tbl (keyword table-name)+ kw-gate (keyword gate-table)+ tbl-model (keyword table-name "model")+ tbl-model-id (keyword table-name "model_id")+ tbl-root-coll-type (keyword table-name "root_collection_type")+ gate-id (keyword gate-table "id")+ gate-doc-root [:->> (keyword gate-table "document") [:inline "root_collection_type"]]+ composite-gate-id [:|| tbl-model [:inline "_"] tbl-model-id]]+ (execute! {:alter-table [kw-tbl] :add-column [[:root_collection_type :text :if-not-exists]]})+ ;; Per-row backfill: take whatever the gate document says — authoritative when present.+ (execute! {:update kw-tbl+ :from [kw-gate]+ :set {:root_collection_type gate-doc-root}+ :where [:and+ [:= gate-id composite-gate-id]+ [:= tbl-root-coll-type nil]+ [:!= gate-doc-root nil]]})+ ;; Forest backfill: one UPDATE per distinct root type, filling rows the gate doc missed.+ (doseq [[root-type entries] (group-by val root-type-by-coll-id)]+ (execute! {:update kw-tbl+ :set {:root_collection_type [:inline root-type]}+ :where [:and+ [:= :root_collection_type nil]+ [:in :collection_id (mapv key entries)]]})))))))+(defn migrate-dynamic-schema!"Migrate runtime-managed schema, ie. schema of `index_table_...` tables. Migration author is responsible for removingleftovers if necessary."@@ -91,5 +150,7 @@;; migration 1: all tables dropped in schema migration in single function call;; migration 2: add personal_owner_id column to index tables;; migration 3: add collection_type and data_layer columns to index tables+ ;; migration 4: add root_collection_type column to index tables(add-personal-owner-id-column! tx index-metadata)- (add-collection-type-and-data-layer-columns! tx index-metadata))+ (add-collection-type-and-data-layer-columns! tx index-metadata)+ (add-root-collection-type-column! tx index-metadata))diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/index.cljindex bef9d7380c2..8b7d29565cc 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj@@ -67,6 +67,7 @@[:pinned :boolean][:verified :boolean][:collection_type :text]+ [:root_collection_type :text][:data_layer :text][:dashboardcard_count :int][:view_count :int]@@ -152,7 +153,8 @@"Convert a document to a database record with a provided embedding."[owner-ids {:keys [model id embedding searchable_text embeddable_text native_query created_at creator_id updated_atlast_editor_id archived verified official_collection database_id collection_id display_type legacy_input- pinned dashboardcard_count view_count last_viewed_at collection_type data_layer] :as doc}]+ pinned dashboardcard_count view_count last_viewed_at collection_type root_collection_type+ data_layer] :as doc}]{:model model:model_id id:collection_id collection_id@@ -168,6 +170,7 @@:pinned (some-> pinned to-boolean):verified (some-> verified to-boolean):collection_type collection_type+ :root_collection_type root_collection_type:data_layer data_layer:dashboardcard_count dashboardcard_count:view_count view_count@@ -284,7 +287,7 @@(let [table-name (or table-name (model-table-name embedding-model))]{:embedding-model embedding-model:table-name table-name- :version 3}))+ :version 4}))(defn- upsert-embedding!-fn [connectable index text->docs](fn [text->embedding]@@ -585,6 +588,7 @@[:pinned :pinned][:verified :verified][:collection_type :collection_type]+ [:root_collection_type :root_collection_type][:data_layer :data_layer][:dashboardcard_count :dashboardcard_count][:view_count :view_count]diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/scoring.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/scoring.cljindex 13468e530e2..2b1e1eacdd0 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/scoring.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/scoring.clj@@ -94,7 +94,9 @@:prefix (if search-string;; in this case, we need to transform the string into a pattern in code, so forced to use helper(search.scoring/prefix [:lower :name] (u/lower-case-en search-string))- [:inline 0])}))+ [:inline 0])+ :library (search.scoring/library-score-expr)+ :data-layer (search.scoring/data-layer-score-expr search-ctx)}))(def ^:private enterprise-scorers{:official-collection {:expr (search.scoring/truthy :official_collection)diff --git a/enterprise/backend/test/metabase_enterprise/search/scoring_test.clj b/enterprise/backend/test/metabase_enterprise/search/scoring_test.cljindex 74252738c21..2cc068f0e44 100644--- a/enterprise/backend/test/metabase_enterprise/search/scoring_test.clj+++ b/enterprise/backend/test/metabase_enterprise/search/scoring_test.clj@@ -37,7 +37,7 @@(mt/with-premium-features #{}(-> (scoring/score-and-result item {:search-string search-string}) :score)))-(deftest official-collection-tests+(deftest official-collection-bumps-value-test(testing "it should bump up the value of items in official collections";; using the ee implementation that isn't wrapped by premium features token check(let [search-string "custom expression examples"@@ -63,12 +63,17 @@"custom expression examples"](mapv :name (sort-by #(oss-score search-string %)(shuffle [a b c d])))))- (is (= ["customer examples of bad sorting"- "customer success stories"+ ;; With :official-collection lifted to 80 in :default, the official bump dominates the+ ;; text scorers — `d` now ranks highest, landing last under the ascending sort, despite+ ;; having the weakest text match.+ (is (= ["customer success stories""examples of custom expressions"- "custom expression examples"]+ "custom expression examples"+ "customer examples of bad sorting"](mapv :name (sort-by #(ee-score search-string %)- (shuffle [a b c (assoc d :collection_authority_level "official")])))))))+ (shuffle [a b c (assoc d :collection_authority_level "official")]))))))))++(deftest verified-items-bump-value-test(testing "It should bump up the value of verified items"(let [ss "foo"a {:name "foobar"diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/migration_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/migration_test.cljindex 23842200c9a..58532ede896 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/migration_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/migration_test.clj@@ -11,13 +11,14 @@[metabase-enterprise.semantic-search.db.migration.impl :as semantic.db.migration.impl][metabase-enterprise.semantic-search.env :as semantic.env][metabase-enterprise.semantic-search.index :as semantic.index]- [metabase-enterprise.semantic-search.pgvector-api :as semantic.pgvector-api][metabase-enterprise.semantic-search.test-util :as semantic.tu]+ [metabase.collections.models.collection :as collection][metabase.test :as mt][metabase.util :as u][metabase.util.log :as log][metabase.util.log.capture :as log.capture]- [next.jdbc :as jdbc]))+ [next.jdbc :as jdbc]+ [next.jdbc.result-set :as jdbc.rs]))(set! *warn-on-reflection* true)@@ -82,8 +83,7 @@original-maybe-migrate @#'semantic.db.migration/maybe-migrate!results (atom {:executed-migrations 0:log []})]- (with-redefs-fn {#'semantic.pgvector-api/index-documents! (constantly nil)- ;; Wrap maybe-migrate! to log timestamps AFTER lock is acquired+ (with-redefs-fn {;; Wrap maybe-migrate! to log timestamps AFTER lock is acquired;; (maybe-migrate! is called inside with-migrate-tx, after the lock)#'semantic.db.migration/maybe-migrate!(fn [& args]@@ -128,92 +128,92 @@(try(mt/with-premium-features #{:semantic-search}(semantic.tu/with-test-db-defaults!- (with-redefs [semantic.pgvector-api/index-documents! (constantly nil)]- (semantic.core/init! (semantic.tu/mock-documents) nil)- (testing "migration table has expected columns"- (is (map-contains-keys?- (jdbc/execute-one! (semantic.env/get-pgvector-datasource!)- (sql/format {:select [:*]- :from [:migration]}))- (qualify :migration [:migrated_at- :status- :version]))))- (testing "control table has expected columns"- (is (map-contains-keys?- (jdbc/execute-one! (semantic.env/get-pgvector-datasource!)- (sql/format {:select [:*]- :from [:index_control]}))- (qualify :index_control [:active_id- :active_updated_at- :id- :version]))))- (testing "metadata table has expected columns"- (is (map-contains-keys?- (jdbc/execute-one! (semantic.env/get-pgvector-datasource!)- (sql/format {:select [:*]- :from [:index_metadata]}))- (qualify :index_metadata [:id- :index_created_at- :index_version- :indexer_last_poll- :indexer_last_seen- :indexer_last_seen_hash- :indexer_last_seen_id- :model_name- :provider- :table_name- :vector_dimensions]))))- (testing "index table has expected columns"- (let [index-table (->- (jdbc/execute-one! (semantic.db.datasource/ensure-initialized-data-source!)- (sql/format {:select [[:im.table_name]]- :from [[:index_control :ic]]- :join [[:index_metadata :im]- [:= :ic.active_id :im.id]]}))- :index_metadata/table_name)]- (is (=? #{"archived"- "collection_id"- "collection_type"- "content"- "created_at"- "creator_id"- "dashboardcard_count"- "database_id"- "data_layer"- "display_type"- "embedding"- "id"- "last_editor_id"- "last_viewed_at"- "legacy_input"- "metadata"- "model"- "model_created_at"- "model_id"- "model_updated_at"- "name"- "official_collection"- "personal_owner_id"- "pinned"- "text_search_vector"- "text_search_with_native_query_vector"- "verified"- "view_count"}- (->> (jdbc/execute! (semantic.env/get-pgvector-datasource!)- (sql/format {:select [:column_name]- :from [:information_schema.columns]- :where [[:= :table_name [:inline index-table]]]}))- (map :columns/column_name)- set)))))- (testing "index table has expected columns"- (is (= ["document" "document_hash" "gated_at" "id" "model" "model_id" "updated_at"]- (->> (jdbc/execute! (semantic.env/get-pgvector-datasource!)- (sql/format {:select [:column_name]- :from [:information_schema.columns]- :where [[:= :table_name [:inline "index_gate"]]]}))- (map :columns/column_name)- sort- vec)))))))+ (semantic.core/init! (semantic.tu/mock-documents) nil)+ (testing "migration table has expected columns"+ (is (map-contains-keys?+ (jdbc/execute-one! (semantic.env/get-pgvector-datasource!)+ (sql/format {:select [:*]+ :from [:migration]}))+ (qualify :migration [:migrated_at+ :status+ :version]))))+ (testing "control table has expected columns"+ (is (map-contains-keys?+ (jdbc/execute-one! (semantic.env/get-pgvector-datasource!)+ (sql/format {:select [:*]+ :from [:index_control]}))+ (qualify :index_control [:active_id+ :active_updated_at+ :id+ :version]))))+ (testing "metadata table has expected columns"+ (is (map-contains-keys?+ (jdbc/execute-one! (semantic.env/get-pgvector-datasource!)+ (sql/format {:select [:*]+ :from [:index_metadata]}))+ (qualify :index_metadata [:id+ :index_created_at+ :index_version+ :indexer_last_poll+ :indexer_last_seen+ :indexer_last_seen_hash+ :indexer_last_seen_id+ :model_name+ :provider+ :table_name+ :vector_dimensions]))))+ (testing "index table has expected columns"+ (let [index-table (->+ (jdbc/execute-one! (semantic.db.datasource/ensure-initialized-data-source!)+ (sql/format {:select [[:im.table_name]]+ :from [[:index_control :ic]]+ :join [[:index_metadata :im]+ [:= :ic.active_id :im.id]]}))+ :index_metadata/table_name)]+ (is (=? #{"archived"+ "collection_id"+ "collection_type"+ "content"+ "created_at"+ "creator_id"+ "dashboardcard_count"+ "database_id"+ "data_layer"+ "display_type"+ "embedding"+ "id"+ "last_editor_id"+ "last_viewed_at"+ "legacy_input"+ "metadata"+ "model"+ "model_created_at"+ "model_id"+ "model_updated_at"+ "name"+ "official_collection"+ "personal_owner_id"+ "pinned"+ "root_collection_type"+ "text_search_vector"+ "text_search_with_native_query_vector"+ "verified"+ "view_count"}+ (->> (jdbc/execute! (semantic.env/get-pgvector-datasource!)+ (sql/format {:select [:column_name]+ :from [:information_schema.columns]+ :where [[:= :table_name [:inline index-table]]]}))+ (map :columns/column_name)+ set)))))+ (testing "index table has expected columns"+ (is (= ["document" "document_hash" "gated_at" "id" "model" "model_id" "updated_at"]+ (->> (jdbc/execute! (semantic.env/get-pgvector-datasource!)+ (sql/format {:select [:column_name]+ :from [:information_schema.columns]+ :where [[:= :table_name [:inline "index_gate"]]]}))+ (map :columns/column_name)+ sort+ vec))))))(catch Throwable e(log/fatal e)(throw e))))@@ -229,42 +229,151 @@(deftest dynamic-schema-migration-test(mt/with-premium-features #{:semantic-search}(semantic.tu/with-test-db-defaults!- (with-redefs [semantic.pgvector-api/index-documents! (constantly nil)]- (semantic.core/init! (semantic.tu/mock-documents) nil)- ;; add column to index table- (let [original-dynamic-schema semantic.db.migration.impl/dynamic-schema-version]- (with-redefs-fn {#'semantic.db.migration.impl/migrate-dynamic-schema!- (fn [tx _opts]- (let [table_names (->> (jdbc/execute! tx- (sql/format {:select [:table_name]- :from [:index_metadata]- :where [[:< :index_version semantic.db.migration.impl/dynamic-schema-version]]- :group-by [:table_name]}))- (map :index_metadata/table_name)- set)]- (doseq [table_name table_names]- (when-not (has-column?! tx table_name "new_col")- (jdbc/execute! tx (sql/format {:alter-table [table_name] :add-column [[:new_col :int]]}))))- (jdbc/execute! tx (sql/format {:update :index_metadata- :set {:index_version semantic.db.migration.impl/dynamic-schema-version}- :where [[:in :table_name table_names]]}))))- #'semantic.db.migration.impl/dynamic-schema-version (inc original-dynamic-schema)}- (fn []- ;; Trigger migration by next initialization attempt- (semantic.core/init! (semantic.tu/mock-documents) nil)- (let [#:index_metadata{:keys [index_version table_name]}- (jdbc/execute-one! (semantic.db.datasource/ensure-initialized-data-source!)- (sql/format- {:select [:*]- :from [:index_metadata]}))]- (testing "Index metadata table ids were updated"- (is (= index_version semantic.db.migration.impl/dynamic-schema-version)))- (testing "Index table contains new column"- (is (has-column?! (semantic.db.datasource/ensure-initialized-data-source!)- table_name "new_col")))))))))))+ (semantic.core/init! (semantic.tu/mock-documents) nil)+ ;; add column to index table+ (let [original-dynamic-schema semantic.db.migration.impl/dynamic-schema-version]+ (with-redefs-fn {#'semantic.db.migration.impl/migrate-dynamic-schema!+ (fn [tx _opts]+ (let [table_names (->> (jdbc/execute! tx+ (sql/format {:select [:table_name]+ :from [:index_metadata]+ :where [[:< :index_version semantic.db.migration.impl/dynamic-schema-version]]+ :group-by [:table_name]}))+ (map :index_metadata/table_name)+ set)]+ (doseq [table_name table_names]+ (when-not (has-column?! tx table_name "new_col")+ (jdbc/execute! tx (sql/format {:alter-table [table_name] :add-column [[:new_col :int]]}))))+ (jdbc/execute! tx (sql/format {:update :index_metadata+ :set {:index_version semantic.db.migration.impl/dynamic-schema-version}+ :where [[:in :table_name table_names]]}))))+ #'semantic.db.migration.impl/dynamic-schema-version (inc original-dynamic-schema)}+ (fn []+ ;; Trigger migration by next initialization attempt+ (semantic.core/init! (semantic.tu/mock-documents) nil)+ (let [#:index_metadata{:keys [index_version table_name]}+ (jdbc/execute-one! (semantic.db.datasource/ensure-initialized-data-source!)+ (sql/format+ {:select [:*]+ :from [:index_metadata]}))]+ (testing "Index metadata table ids were updated"+ (is (= index_version semantic.db.migration.impl/dynamic-schema-version)))+ (testing "Index table contains new column"+ (is (has-column?! (semantic.db.datasource/ensure-initialized-data-source!)+ table_name "new_col"))))))))))(deftest schema-version-match-default-version-test(testing "Default schema should match dynamic schema version"(is (= semantic.db.migration.impl/dynamic-schema-version(-> (semantic.index/default-index semantic.tu/mock-embedding-model):version)))))++(defn- active-index-table-name!+ [pgvector index-metadata]+ (let [{:keys [control-table-name metadata-table-name]} index-metadata]+ (->> (jdbc/execute-one! pgvector+ (sql/format {:select [:im.table_name]+ :from [[(keyword metadata-table-name) :im]]+ :join [[(keyword control-table-name) :ic] [:= :ic.active_id :im.id]]}))+ vals+ first)))++(defn- insert-index-row!+ "Raw INSERT into the active index table for backfill tests. Fills in the minimum NOT NULL+ columns with placeholder values; `row` supplies anything else (model, model_id,+ collection_type, root_collection_type)."+ [pgvector kw-tbl row]+ (jdbc/execute! pgvector+ (sql/format {:insert-into kw-tbl+ :values [(merge {:name [:inline ""]+ :content [:inline ""]+ :text_search_vector [:to_tsvector [:inline "simple"] [:inline ""]]+ :text_search_with_native_query_vector [:to_tsvector [:inline "simple"] [:inline ""]]+ :embedding [:raw "'[0,0,0,0]'::vector"]}+ row)]}+ :quoted true)))++(deftest migration-4-backfill-test+ (testing "migration 4 backfills root_collection_type from the gate doc, then via an appdb walk of the Library forest"+ (mt/with-premium-features #{:semantic-search}+ ;; :mock-initialized puts the 4-dim mock embedding model in scope so the placeholder+ ;; `[0,0,0,0]` embedding in [[insert-index-row!]] matches the index column's dimensions.+ (semantic.tu/with-test-db! {:mode :mock-initialized}+ ;; Stand up a Library tree: library root → library-data sub → library-data sub-sub,+ ;; plus a library-metrics sub under the root. The backfill should resolve every nested+ ;; collection's `root_collection_type` to the top-level Library's own `:type`.+ (mt/with-temp [:model/Collection {lib-id :id} {:name "Library" :type collection/library-collection-type :location "/"}+ :model/Collection {data-id :id} {:name "Data"+ :type collection/library-data-collection-type+ :location (str "/" lib-id "/")}+ :model/Collection {nested-data-id :id} {:name "Nested Data"+ :type collection/library-data-collection-type+ :location (str "/" lib-id "/" data-id "/")}+ :model/Collection {metrics-id :id} {:name "Metrics"+ :type collection/library-metrics-collection-type+ :location (str "/" lib-id "/")}+ :model/Collection {other-id :id} {:name "Regular" :location "/"}]+ (let [pgvector (semantic.env/get-pgvector-datasource!)+ index-metadata (semantic.env/get-index-metadata)+ gate-tbl (keyword (:gate-table-name index-metadata))+ meta-tbl (keyword (:metadata-table-name index-metadata))+ kw-tbl (keyword (active-index-table-name! pgvector index-metadata))]+ ;; row 1: NULL root_collection_type; matching gate doc carries "library" in its document JSON+ (insert-index-row! pgvector kw-tbl {:model [:inline "card"] :model_id [:inline "1"]})+ ;; row 2: collection_id = top-level Library (root of its own tree)+ (insert-index-row! pgvector kw-tbl {:model [:inline "card"]+ :model_id [:inline "2"]+ :collection_id lib-id})+ ;; row 3: collection_id = first-level library-data sub-collection+ (insert-index-row! pgvector kw-tbl {:model [:inline "card"]+ :model_id [:inline "3"]+ :collection_id data-id})+ ;; row 4: collection_id = deeper library-data sub-sub-collection+ (insert-index-row! pgvector kw-tbl {:model [:inline "card"]+ :model_id [:inline "4"]+ :collection_id nested-data-id})+ ;; row 5: collection_id = library-metrics sub-collection — still resolves to "library"+ (insert-index-row! pgvector kw-tbl {:model [:inline "card"]+ :model_id [:inline "5"]+ :collection_id metrics-id})+ ;; row 6: collection_id = a non-library collection — backfill must leave NULL+ (insert-index-row! pgvector kw-tbl {:model [:inline "card"]+ :model_id [:inline "6"]+ :collection_id other-id})+ ;; row 7: already-populated root_collection_type — backfill must leave it alone+ (insert-index-row! pgvector kw-tbl {:model [:inline "card"]+ :model_id [:inline "7"]+ :root_collection_type [:inline "library-data"]})+ ;; Gate doc for row 1 — JSON includes root_collection_type+ (jdbc/execute! pgvector+ (sql/format {:insert-into gate-tbl+ :values [{:id [:inline "card_1"]+ :model [:inline "card"]+ :model_id [:inline "1"]+ :updated_at [:now]+ :document [:cast [:inline "{\"root_collection_type\":\"library\"}"] :jsonb]+ :document_hash [:inline "h"]}]}+ :quoted true))+ ;; Roll metadata.index_version back to 3 so migration 4 re-runs against the table.+ (jdbc/execute! pgvector+ (sql/format {:update meta-tbl+ :set {:index_version 3}}))+ ;; Trigger re-migration via init.+ (semantic.core/init! (semantic.tu/mock-documents) nil)+ ;; Verify each backfill branch.+ (let [rows-by-id (->> (jdbc/execute! pgvector+ (sql/format {:select [:model_id :root_collection_type]+ :from [kw-tbl]+ :order-by [:model_id]}+ :quoted true)+ {:builder-fn jdbc.rs/as-unqualified-maps})+ (map (juxt :model_id :root_collection_type))+ (into {}))]+ (is (= {"1" "library" ; pulled from gate.document->>'root_collection_type'+ "2" "library" ; collection_id IS the library root+ "3" "library" ; library-data sub-collection — walks up to library+ "4" "library" ; deeper library-data sub-sub — still walks up+ "5" "library" ; library-metrics sub-collection — still walks up+ "6" nil ; non-library collection, no gate doc+ "7" "library-data"} ; pre-existing value preserved+ rows-by-id)))))))))diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/scoring_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/scoring_test.cljindex df309a73149..13e39e2644a 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/scoring_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/scoring_test.clj@@ -1,6 +1,7 @@(ns metabase-enterprise.semantic-search.scoring-test(:require[clojure.core.memoize :as memoize]+ [clojure.string :as str][clojure.test :refer :all][metabase-enterprise.semantic-search.index :as semantic.index][metabase-enterprise.semantic-search.scoring :as semantic.scoring]@@ -235,6 +236,56 @@["collection" 2 "collection official"]](search-results* "collection")))))))+(deftest library-test+ (mt/with-premium-features #{:semantic-search}+ (testing "Items whose root ancestor collection is a library type rank above non-library items"+ (with-index-contents!+ [{:model "card" :id 1 :name "plain card"}+ {:model "card" :id 2 :name "lib-tree card library" :root_collection_type "library"}+ {:model "card" :id 3 :name "lib-tree card library-data" :root_collection_type "library-data"}+ {:model "card" :id 4 :name "lib-tree card library-metrics" :root_collection_type "library-metrics"}+ {:model "card" :id 5 :name "trashed card" :root_collection_type "trash"}]+ (let [in-library? (fn [[_ _ nm]] (str/includes? nm "lib-tree"))]+ (testing "with positive :library weight, library items come first"+ (is (= [true true true false false]+ (map in-library? (semantic.tu/with-weights {:library 1} (search-results* "card"))))))+ (testing "with negative :library weight, library items come last"+ (is (= [false false true true true]+ (map in-library? (semantic.tu/with-weights {:library -1} (search-results* "card")))))))))))++(deftest data-layer-test+ (mt/with-premium-features #{:semantic-search}+ (testing ":data-layer scorer reads the active per-tier weight via :data-layer/* params"+ (with-index-contents!+ [{:model "table" :id 1 :name "table no layer"}+ {:model "table" :id 2 :name "table final" :data_layer "final"}+ {:model "table" :id 3 :name "table internal" :data_layer "internal"}+ {:model "table" :id 4 :name "table hidden" :data_layer "hidden"}]+ (testing "final tables lead when only :data-layer/final is positive"+ (is (= 2 (-> (semantic.tu/with-weights {:data-layer 1 :data-layer/final 1}+ (search-results* "table"))+ first second))))+ (testing "internal tables lead when only :data-layer/internal is positive"+ (is (= 3 (-> (semantic.tu/with-weights {:data-layer 1 :data-layer/internal 1}+ (search-results* "table"))+ first second))))+ (testing "hidden tables lead when only :data-layer/hidden is positive"+ (is (= 4 (-> (semantic.tu/with-weights {:data-layer 1 :data-layer/hidden 1}+ (search-results* "table"))+ first second))))))+ (testing "tier ordering: final > internal > hidden under :metabot magnitudes"+ (with-index-contents!+ [{:model "table" :id 1 :name "foo table final" :data_layer "final"}+ {:model "table" :id 2 :name "foo table internal" :data_layer "internal"}+ {:model "table" :id 3 :name "foo table hidden" :data_layer "hidden"}]+ (is (= [1 2 3]+ (->> (semantic.tu/with-weights {:data-layer 33+ :data-layer/final 1+ :data-layer/internal 0.3+ :data-layer/hidden 0.03}+ (search-results* "foo table"))+ (map second))))))))+(deftest verified-test(with-index-contents)diff --git a/src/metabase/lib/schema/expression.cljc b/src/metabase/lib/schema/expression.cljcindex 7331731553b..f4e8611b545 100644--- a/src/metabase/lib/schema/expression.cljc+++ b/src/metabase/lib/schema/expression.cljc@@ -266,12 +266,9 @@#(cond(vector? %) ;; non-literal (checked above)true-(not (int? %))false-(not (pos? %))false-:elsetrue)]])diff --git a/src/metabase/search/appdb/scoring.clj b/src/metabase/search/appdb/scoring.cljindex e053e956575..b8a289b907b 100644--- a/src/metabase/search/appdb/scoring.clj+++ b/src/metabase/search/appdb/scoring.clj@@ -2,7 +2,6 @@(:require[clojure.core.memoize :as memoize][honey.sql.helpers :as sql.helpers]- [metabase.collections.models.collection :as collection][metabase.config.core :as config][metabase.premium-features.core :refer [defenterprise]][metabase.search.appdb.index :as search.index]@@ -64,26 +63,8 @@;; in this case, we need to transform the string into a pattern in code, so forced to use helper(search.scoring/prefix [:lower :search_index.name] (u/lower-case-en search-string))[:inline 0])- ;; :library matches items whose root (top-level) ancestor collection is one of the library types.- ;; The :root-collection-type attr is computed at ingestion time by walking the collection's- ;; materialized path (see metabase.collections.models.collection/root-collection-type), so items- ;; in arbitrarily deep sub-collections of a library tree still match here.- :library [:case- (into [:or]- (for [t (sort collection/library-collection-types)]- [:= :search_index.root_collection_type [:inline t]]))- [:inline 1]- :else [:inline 0]]- ;; :data-layer mirrors the :model/* pattern — one scorer, per-tier weights live under :data-layer/*- ;; (see metabase.search.config/scorer-param). Final/internal/hidden are mutually exclusive.- :data-layer [:case- [:= :search_index.data_layer [:inline "final"]]- [:inline (or (search.config/scorer-param search-ctx :data-layer :final) 0)]- [:= :search_index.data_layer [:inline "internal"]]- [:inline (or (search.config/scorer-param search-ctx :data-layer :internal) 0)]- [:= :search_index.data_layer [:inline "hidden"]]- [:inline (or (search.config/scorer-param search-ctx :data-layer :hidden) 0)]- :else [:inline 0]]}))+ :library (search.scoring/library-score-expr)+ :data-layer (search.scoring/data-layer-score-expr search-ctx)}))(defenterprise scorers"Return the select-item expressions used to calculate the score for each search result."diff --git a/src/metabase/search/config.clj b/src/metabase/search/config.cljindex c1316ce186c..d2f02cb054f 100644--- a/src/metabase/search/config.clj+++ b/src/metabase/search/config.clj@@ -84,13 +84,18 @@:user-recency 5:dashboard 0:model 2- :official-collection 1- :verified 1:view-count 2:text 5:mine 1:exact 5:prefix 0+ ;; User-curated tiers, from strongest to weakest.+ ;; :library at 200 exceeds :official-collection + :verified (80 each) combined,+ ;; so a library item always outranks a non-library one.+ ;; Base text/recency scorers (0–5 range) only break ties *within* a tier.+ :library 200+ :official-collection 80+ :verified 80;; RRF is the "Reciprocal Rank Fusion" score used by the semantic search backend to blend semantic and keyword scores:rrf 500}:command-palette@@ -108,14 +113,28 @@:model/dataset 1:model/metric 1:model/question 0}+ ;; TODO: lift :data-layer up to :default. It's a structural signal (every visible warehouse+ ;; table gets `data_layer "final"`), so the +33 boost flips orderings across every search+ ;; surface and breaks e2e specs that pin specific top results. To make progress:+ ;;+ ;; - Update the e2e snapshot in `e2e/snapshot-creators/default.cy.snap.js` so the sample-DB+ ;; tables aren't all at `final` (mark non-essential ones `internal`/`hidden`), or+ ;; - Rewrite the brittle assertions to not depend on a specific top result.+ ;;+ ;; Affected specs (broken by an earlier lift attempt):+ ;;+ ;; - `search.cy.spec.js`+ ;; - `models.cy.spec.js`+ ;; - `documents.cy.spec.ts`+ ;; - `document-links.cy.spec.ts`+ ;; - `custom-viz.cy.spec.ts`+ ;; - `search-snowplow.cy.spec.js`:metabot- {:library 100- :official-collection 80- :verified 80- :data-layer 1 ; overall multiplier; per-tier weights live under :data-layer/* below- :data-layer/final 33- :data-layer/internal 10- :data-layer/hidden 1}})+ {:data-layer 33+ :data-layer/final 1 ; ≈ 33+ :data-layer/internal 0.3 ; ≈ 10+ :data-layer/hidden 0.03 ; ≈ 1+ }})(def ^:private FilterDef"A relaxed definition, capturing how we can write the filter - with some fields omitted."diff --git a/src/metabase/search/scoring.clj b/src/metabase/search/scoring.cljindex 5ecc47a4348..8d8f1a523f6 100644--- a/src/metabase/search/scoring.clj+++ b/src/metabase/search/scoring.clj@@ -3,6 +3,7 @@[clojure.string :as str][honey.sql.helpers :as sql.helpers][metabase.app-db.core :as mdb]+ [metabase.collections.models.collection :as collection][metabase.search.config :as search.config][metabase.util.honey-sql-2 :as h2x]))@@ -84,6 +85,29 @@[:= :search_index.model [:inline "metric"]] [:inline "card"]:else :search_index.model]]]}))+(defn library-score-expr+ "Score expression: 1 when `:root_collection_type` is one of the library collection types, else 0."+ []+ ;; `:root-collection-type` is set at ingestion via `collection/root-collection-type` (walks the+ ;; materialized path), so deeply nested items in a library tree still match.+ [:case+ (into [:or]+ (for [t (sort collection/library-collection-types)]+ [:= :root_collection_type [:inline t]]))+ [:inline 1]+ :else [:inline 0]])++(defn data-layer-score-expr+ "Score expression: per-tier weight when `:data_layer` is `final`/`internal`/`hidden`, else 0.+ Per-tier weights are read from `:data-layer/*` keys via [[search.config/scorer-param]]."+ [search-ctx]+ (let [tier-weight #(or (search.config/scorer-param search-ctx :data-layer %) 0)]+ [:case+ [:= :data_layer [:inline "final"]] [:inline (tier-weight :final)]+ [:= :data_layer [:inline "internal"]] [:inline (tier-weight :internal)]+ [:= :data_layer [:inline "hidden"]] [:inline (tier-weight :hidden)]+ :else [:inline 0]]))+(defn model-rank-expr"Score an item based on its :model type."[search-ctx]diff --git a/test/metabase/search/appdb/scoring_test.clj b/test/metabase/search/appdb/scoring_test.cljindex fb0f50bbe9e..cff8ab9c3dc 100644--- a/test/metabase/search/appdb/scoring_test.clj+++ b/test/metabase/search/appdb/scoring_test.clj@@ -2,6 +2,7 @@(:require[clojure.core.memoize :as memoize][clojure.set :as set]+ [clojure.string :as str][clojure.test :refer :all][metabase.app-db.core :as mdb][metabase.search.appdb.index :as search.index]@@ -326,15 +327,14 @@:model/Collection sub-sub {:name "sub of sub" :location (format "/%d/%d/" (:id lib) (:id sub))}:model/Collection other {:name "non-library" :location "/"}](with-index-contents- [{:model "card" :id 1 :name "card plain" :collection_id (:id other) :collection_location (:location other)}- {:model "card" :id 2 :name "card in library" :collection_id (:id lib) :collection_location (:location lib) :collection_type "library"}- {:model "card" :id 3 :name "card in library-data" :collection_id (:id lib-data) :collection_location (:location lib-data) :collection_type "library-data"}- {:model "card" :id 4 :name "card in lib-metrics" :collection_id (:id lib-met) :collection_location (:location lib-met) :collection_type "library-metrics"}- {:model "card" :id 5 :name "card in sub of lib" :collection_id (:id sub) :collection_location (:location sub)}- {:model "card" :id 6 :name "card in sub of sub" :collection_id (:id sub-sub) :collection_location (:location sub-sub)}- {:model "card" :id 7 :name "card in trash" :collection_type "trash"}]- (let [library-id? #{2 3 4 5 6}- in-library? (fn [[_ id _]] (boolean (library-id? id)))]+ [{:model "card" :id 1 :name "plain card" :collection_id (:id other) :collection_location (:location other)}+ {:model "card" :id 2 :name "lib-tree card library" :collection_id (:id lib) :collection_location (:location lib) :collection_type "library"}+ {:model "card" :id 3 :name "lib-tree card library-data" :collection_id (:id lib-data) :collection_location (:location lib-data) :collection_type "library-data"}+ {:model "card" :id 4 :name "lib-tree card library-metrics" :collection_id (:id lib-met) :collection_location (:location lib-met) :collection_type "library-metrics"}+ {:model "card" :id 5 :name "lib-tree card sub" :collection_id (:id sub) :collection_location (:location sub)}+ {:model "card" :id 6 :name "lib-tree card sub-sub" :collection_id (:id sub-sub) :collection_location (:location sub-sub)}+ {:model "card" :id 7 :name "trashed card" :collection_type "trash"}]+ (let [in-library? (fn [[_ _ nm]] (str/includes? nm "lib-tree"))](testing "with positive :library weight, items inside library trees come first"(is (= [true true true true true false false](map in-library? (with-weights {:library 1} (search-results* "card"))))))@@ -361,15 +361,15 @@(is (= 4 (-> (with-weights {:data-layer 1 :data-layer/hidden 1}(search-results* "table"))first second))))))- (testing "Metabot tier ordering: final > internal > hidden when all weights active"+ (testing "tier ordering: final > internal > hidden under :metabot magnitudes"(with-index-contents- [{:model "table" :id 1 :name "metabot table final" :data_layer "final"}- {:model "table" :id 2 :name "metabot table internal" :data_layer "internal"}- {:model "table" :id 3 :name "metabot table hidden" :data_layer "hidden"}]+ [{:model "table" :id 1 :name "foo table final" :data_layer "final"}+ {:model "table" :id 2 :name "foo table internal" :data_layer "internal"}+ {:model "table" :id 3 :name "foo table hidden" :data_layer "hidden"}](is (= [1 2 3]- (->> (with-weights {:data-layer 1- :data-layer/final 33- :data-layer/internal 10- :data-layer/hidden 1}- (search-results* "metabot table"))+ (->> (with-weights {:data-layer 33+ :data-layer/final 1+ :data-layer/internal 0.3+ :data-layer/hidden 0.03}+ (search-results* "foo table"))(map second)))))))diff --git a/test/metabase/search/config_test.clj b/test/metabase/search/config_test.cljindex c2ce587c9d3..a258ee06bba 100644--- a/test/metabase/search/config_test.clj+++ b/test/metabase/search/config_test.clj@@ -22,17 +22,3 @@(search.config/filter-default :search.engine/in-place :command-palette :filter-items-in-personal-collection)))(is (= "exclude-others"(search.config/filter-default :search.engine/in-place :search-app :filter-items-in-personal-collection)))))--(deftest metabot-weights-test- (testing "the :metabot context defines the tuned curation tier weights"- (is (=? {:library 100- :official-collection 80- :verified 80- :data-layer 1- :data-layer/final 33- :data-layer/internal 10- :data-layer/hidden 1}- (search.config/weights {:context :metabot}))))- (testing "request-level :weights override beats :metabot static weights"- (is (= 7 (-> (search.config/weights {:context :metabot :weights {:library 7}})- :library)))))