Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Boost library-collection items in search ranking (#74806)

Chris Truterd587fb7ffee1
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration/impl.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration/impl.clj
index c513c26b7c0..a1bdbe7b85b 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration/impl.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration/impl.clj
@@ -2,7 +2,10 @@
(:require
[honey.sql :as sql]
[metabase-enterprise.semantic-search.index-metadata :as semantic.index-metadata]
- [next.jdbc :as jdbc]))
+ [metabase.util :as u]
+ [metabase.util.log :as log]
+ [next.jdbc :as jdbc]
+ [toucan2.core :as t2]))
(def schema-version
"Version to compare the [[metabase-enterprise.semantic-search.db.migration/db-version]] with. If this is higher,
@@ -37,7 +40,7 @@
(def dynamic-schema-version
"Code version of dynamic schema (index_table_xyzs). If higher than what's found in db dynamic schema migration will
be attempted."
- 3)
+ 4)
(defn- alter-index-tables!
"Run `alter-fn` against each existing index table whose `index_version` is below `target-version`, then bump those
@@ -74,8 +77,10 @@
:add-column [[:personal_owner_id :int :if-not-exists]]}))))
(defn- add-collection-type-and-data-layer-columns!
- "Migration 3: Add `collection_type` and `data_layer` columns to index tables for the new ranking signals
- (`:library` on `collection_type`; `:data-layer` per-tier weights on `data_layer`)."
+ "Migration 3: Add `collection_type` and `data_layer` columns to index tables.
+ `collection_type` is surfaced for downstream consumers (e.g. `metabase.search.impl/serialize`);
+ `data_layer` powers the `:data-layer` scorer (per-tier weights under `:data-layer/*`).
+ The `:library` scorer was reworked in migration 4 to read `root_collection_type` instead."
[tx index-metadata]
(alter-index-tables! tx index-metadata 3
(fn [execute! table-name]
@@ -84,6 +89,60 @@
(execute! {:alter-table [(keyword table-name)]
:add-column [[:data_layer :text :if-not-exists]]}))))
+(defn- library-root-type-by-collection-id
+ "Map `collection-id → root-collection-type` for every collection in a Library tree.
+ Returns `{}` and logs a warning if the appdb lookup fails."
+ [library-types]
+ ;; Catch covers test setups that exercise pgvector before the appdb schema is up;
+ ;; production semantic-search init always runs after appdb migration.
+ (try
+ (u/for-map [{root-id :id root-type :type} (t2/select [:model/Collection :id :type]
+ :type [:in library-types]
+ :location "/")
+ coll-id (cons root-id (t2/select-pks-set :model/Collection
+ :location [:like (str "/" root-id "/%")]))]
+ [coll-id root-type])
+ (catch Exception e
+ (log/warn e "Skipping Library forest backfill — appdb lookup failed")
+ {})))
+
+(defn- add-root-collection-type-column!
+ "Migration 4: add `root_collection_type` to index tables, backfilling first from each row's
+ gate document and then from an appdb sweep of the Library forest. Drives the `:library` scorer."
+ [tx index-metadata]
+ ;; Library types are hardcoded: migrations are frozen snapshots, and semantic-search doesn't `:use` `collections`.
+ (let [gate-table (:gate-table-name index-metadata)
+ library-types ["library" "library-data" "library-metrics"]
+ ;; Resolved once — the same map applies to every index table.
+ root-type-by-coll-id (library-root-type-by-collection-id library-types)]
+ (alter-index-tables!
+ tx index-metadata 4
+ (fn [execute! table-name]
+ (let [kw-tbl (keyword table-name)
+ kw-gate (keyword gate-table)
+ tbl-model (keyword table-name "model")
+ tbl-model-id (keyword table-name "model_id")
+ tbl-root-coll-type (keyword table-name "root_collection_type")
+ gate-id (keyword gate-table "id")
+ gate-doc-root [:->> (keyword gate-table "document") [:inline "root_collection_type"]]
+ composite-gate-id [:|| tbl-model [:inline "_"] tbl-model-id]]
+ (execute! {:alter-table [kw-tbl] :add-column [[:root_collection_type :text :if-not-exists]]})
+ ;; Per-row backfill: take whatever the gate document says — authoritative when present.
+ (execute! {:update kw-tbl
+ :from [kw-gate]
+ :set {:root_collection_type gate-doc-root}
+ :where [:and
+ [:= gate-id composite-gate-id]
+ [:= tbl-root-coll-type nil]
+ [:!= gate-doc-root nil]]})
+ ;; Forest backfill: one UPDATE per distinct root type, filling rows the gate doc missed.
+ (doseq [[root-type entries] (group-by val root-type-by-coll-id)]
+ (execute! {:update kw-tbl
+ :set {:root_collection_type [:inline root-type]}
+ :where [:and
+ [:= :root_collection_type nil]
+ [:in :collection_id (mapv key entries)]]})))))))
+
(defn migrate-dynamic-schema!
"Migrate runtime-managed schema, ie. schema of `index_table_...` tables. Migration author is responsible for removing
leftovers if necessary."
@@ -91,5 +150,7 @@
;; migration 1: all tables dropped in schema migration in single function call
;; migration 2: add personal_owner_id column to index tables
;; migration 3: add collection_type and data_layer columns to index tables
+ ;; migration 4: add root_collection_type column to index tables
(add-personal-owner-id-column! tx index-metadata)
- (add-collection-type-and-data-layer-columns! tx index-metadata))
+ (add-collection-type-and-data-layer-columns! tx index-metadata)
+ (add-root-collection-type-column! tx index-metadata))
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj
index bef9d7380c2..8b7d29565cc 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj
@@ -67,6 +67,7 @@
[:pinned :boolean]
[:verified :boolean]
[:collection_type :text]
+ [:root_collection_type :text]
[:data_layer :text]
[:dashboardcard_count :int]
[:view_count :int]
@@ -152,7 +153,8 @@
"Convert a document to a database record with a provided embedding."
[owner-ids {:keys [model id embedding searchable_text embeddable_text native_query created_at creator_id updated_at
last_editor_id archived verified official_collection database_id collection_id display_type legacy_input
- pinned dashboardcard_count view_count last_viewed_at collection_type data_layer] :as doc}]
+ pinned dashboardcard_count view_count last_viewed_at collection_type root_collection_type
+ data_layer] :as doc}]
{:model model
:model_id id
:collection_id collection_id
@@ -168,6 +170,7 @@
:pinned (some-> pinned to-boolean)
:verified (some-> verified to-boolean)
:collection_type collection_type
+ :root_collection_type root_collection_type
:data_layer data_layer
:dashboardcard_count dashboardcard_count
:view_count view_count
@@ -284,7 +287,7 @@
(let [table-name (or table-name (model-table-name embedding-model))]
{:embedding-model embedding-model
:table-name table-name
- :version 3}))
+ :version 4}))
(defn- upsert-embedding!-fn [connectable index text->docs]
(fn [text->embedding]
@@ -585,6 +588,7 @@
[:pinned :pinned]
[:verified :verified]
[:collection_type :collection_type]
+ [:root_collection_type :root_collection_type]
[:data_layer :data_layer]
[:dashboardcard_count :dashboardcard_count]
[:view_count :view_count]
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/scoring.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/scoring.clj
index 13468e530e2..2b1e1eacdd0 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/scoring.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/scoring.clj
@@ -94,7 +94,9 @@
:prefix (if search-string
;; in this case, we need to transform the string into a pattern in code, so forced to use helper
(search.scoring/prefix [:lower :name] (u/lower-case-en search-string))
- [:inline 0])}))
+ [:inline 0])
+ :library (search.scoring/library-score-expr)
+ :data-layer (search.scoring/data-layer-score-expr search-ctx)}))
(def ^:private enterprise-scorers
{:official-collection {:expr (search.scoring/truthy :official_collection)
diff --git a/enterprise/backend/test/metabase_enterprise/search/scoring_test.clj b/enterprise/backend/test/metabase_enterprise/search/scoring_test.clj
index 74252738c21..2cc068f0e44 100644
--- a/enterprise/backend/test/metabase_enterprise/search/scoring_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/search/scoring_test.clj
@@ -37,7 +37,7 @@
(mt/with-premium-features #{}
(-> (scoring/score-and-result item {:search-string search-string}) :score)))
-(deftest official-collection-tests
+(deftest official-collection-bumps-value-test
(testing "it should bump up the value of items in official collections"
;; using the ee implementation that isn't wrapped by premium features token check
(let [search-string "custom expression examples"
@@ -63,12 +63,17 @@
"custom expression examples"]
(mapv :name (sort-by #(oss-score search-string %)
(shuffle [a b c d])))))
- (is (= ["customer examples of bad sorting"
- "customer success stories"
+ ;; With :official-collection lifted to 80 in :default, the official bump dominates the
+ ;; text scorers — `d` now ranks highest, landing last under the ascending sort, despite
+ ;; having the weakest text match.
+ (is (= ["customer success stories"
"examples of custom expressions"
- "custom expression examples"]
+ "custom expression examples"
+ "customer examples of bad sorting"]
(mapv :name (sort-by #(ee-score search-string %)
- (shuffle [a b c (assoc d :collection_authority_level "official")])))))))
+ (shuffle [a b c (assoc d :collection_authority_level "official")]))))))))
+
+(deftest verified-items-bump-value-test
(testing "It should bump up the value of verified items"
(let [ss "foo"
a {:name "foobar"
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/migration_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/migration_test.clj
index 23842200c9a..58532ede896 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/migration_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/migration_test.clj
@@ -11,13 +11,14 @@
[metabase-enterprise.semantic-search.db.migration.impl :as semantic.db.migration.impl]
[metabase-enterprise.semantic-search.env :as semantic.env]
[metabase-enterprise.semantic-search.index :as semantic.index]
- [metabase-enterprise.semantic-search.pgvector-api :as semantic.pgvector-api]
[metabase-enterprise.semantic-search.test-util :as semantic.tu]
+ [metabase.collections.models.collection :as collection]
[metabase.test :as mt]
[metabase.util :as u]
[metabase.util.log :as log]
[metabase.util.log.capture :as log.capture]
- [next.jdbc :as jdbc]))
+ [next.jdbc :as jdbc]
+ [next.jdbc.result-set :as jdbc.rs]))
(set! *warn-on-reflection* true)
@@ -82,8 +83,7 @@
original-maybe-migrate @#'semantic.db.migration/maybe-migrate!
results (atom {:executed-migrations 0
:log []})]
- (with-redefs-fn {#'semantic.pgvector-api/index-documents! (constantly nil)
- ;; Wrap maybe-migrate! to log timestamps AFTER lock is acquired
+ (with-redefs-fn {;; Wrap maybe-migrate! to log timestamps AFTER lock is acquired
;; (maybe-migrate! is called inside with-migrate-tx, after the lock)
#'semantic.db.migration/maybe-migrate!
(fn [& args]
@@ -128,92 +128,92 @@
(try
(mt/with-premium-features #{:semantic-search}
(semantic.tu/with-test-db-defaults!
- (with-redefs [semantic.pgvector-api/index-documents! (constantly nil)]
- (semantic.core/init! (semantic.tu/mock-documents) nil)
- (testing "migration table has expected columns"
- (is (map-contains-keys?
- (jdbc/execute-one! (semantic.env/get-pgvector-datasource!)
- (sql/format {:select [:*]
- :from [:migration]}))
- (qualify :migration [:migrated_at
- :status
- :version]))))
- (testing "control table has expected columns"
- (is (map-contains-keys?
- (jdbc/execute-one! (semantic.env/get-pgvector-datasource!)
- (sql/format {:select [:*]
- :from [:index_control]}))
- (qualify :index_control [:active_id
- :active_updated_at
- :id
- :version]))))
- (testing "metadata table has expected columns"
- (is (map-contains-keys?
- (jdbc/execute-one! (semantic.env/get-pgvector-datasource!)
- (sql/format {:select [:*]
- :from [:index_metadata]}))
- (qualify :index_metadata [:id
- :index_created_at
- :index_version
- :indexer_last_poll
- :indexer_last_seen
- :indexer_last_seen_hash
- :indexer_last_seen_id
- :model_name
- :provider
- :table_name
- :vector_dimensions]))))
- (testing "index table has expected columns"
- (let [index-table (->
- (jdbc/execute-one! (semantic.db.datasource/ensure-initialized-data-source!)
- (sql/format {:select [[:im.table_name]]
- :from [[:index_control :ic]]
- :join [[:index_metadata :im]
- [:= :ic.active_id :im.id]]}))
- :index_metadata/table_name)]
- (is (=? #{"archived"
- "collection_id"
- "collection_type"
- "content"
- "created_at"
- "creator_id"
- "dashboardcard_count"
- "database_id"
- "data_layer"
- "display_type"
- "embedding"
- "id"
- "last_editor_id"
- "last_viewed_at"
- "legacy_input"
- "metadata"
- "model"
- "model_created_at"
- "model_id"
- "model_updated_at"
- "name"
- "official_collection"
- "personal_owner_id"
- "pinned"
- "text_search_vector"
- "text_search_with_native_query_vector"
- "verified"
- "view_count"}
- (->> (jdbc/execute! (semantic.env/get-pgvector-datasource!)
- (sql/format {:select [:column_name]
- :from [:information_schema.columns]
- :where [[:= :table_name [:inline index-table]]]}))
- (map :columns/column_name)
- set)))))
- (testing "index table has expected columns"
- (is (= ["document" "document_hash" "gated_at" "id" "model" "model_id" "updated_at"]
- (->> (jdbc/execute! (semantic.env/get-pgvector-datasource!)
- (sql/format {:select [:column_name]
- :from [:information_schema.columns]
- :where [[:= :table_name [:inline "index_gate"]]]}))
- (map :columns/column_name)
- sort
- vec)))))))
+ (semantic.core/init! (semantic.tu/mock-documents) nil)
+ (testing "migration table has expected columns"
+ (is (map-contains-keys?
+ (jdbc/execute-one! (semantic.env/get-pgvector-datasource!)
+ (sql/format {:select [:*]
+ :from [:migration]}))
+ (qualify :migration [:migrated_at
+ :status
+ :version]))))
+ (testing "control table has expected columns"
+ (is (map-contains-keys?
+ (jdbc/execute-one! (semantic.env/get-pgvector-datasource!)
+ (sql/format {:select [:*]
+ :from [:index_control]}))
+ (qualify :index_control [:active_id
+ :active_updated_at
+ :id
+ :version]))))
+ (testing "metadata table has expected columns"
+ (is (map-contains-keys?
+ (jdbc/execute-one! (semantic.env/get-pgvector-datasource!)
+ (sql/format {:select [:*]
+ :from [:index_metadata]}))
+ (qualify :index_metadata [:id
+ :index_created_at
+ :index_version
+ :indexer_last_poll
+ :indexer_last_seen
+ :indexer_last_seen_hash
+ :indexer_last_seen_id
+ :model_name
+ :provider
+ :table_name
+ :vector_dimensions]))))
+ (testing "index table has expected columns"
+ (let [index-table (->
+ (jdbc/execute-one! (semantic.db.datasource/ensure-initialized-data-source!)
+ (sql/format {:select [[:im.table_name]]
+ :from [[:index_control :ic]]
+ :join [[:index_metadata :im]
+ [:= :ic.active_id :im.id]]}))
+ :index_metadata/table_name)]
+ (is (=? #{"archived"
+ "collection_id"
+ "collection_type"
+ "content"
+ "created_at"
+ "creator_id"
+ "dashboardcard_count"
+ "database_id"
+ "data_layer"
+ "display_type"
+ "embedding"
+ "id"
+ "last_editor_id"
+ "last_viewed_at"
+ "legacy_input"
+ "metadata"
+ "model"
+ "model_created_at"
+ "model_id"
+ "model_updated_at"
+ "name"
+ "official_collection"
+ "personal_owner_id"
+ "pinned"
+ "root_collection_type"
+ "text_search_vector"
+ "text_search_with_native_query_vector"
+ "verified"
+ "view_count"}
+ (->> (jdbc/execute! (semantic.env/get-pgvector-datasource!)
+ (sql/format {:select [:column_name]
+ :from [:information_schema.columns]
+ :where [[:= :table_name [:inline index-table]]]}))
+ (map :columns/column_name)
+ set)))))
+ (testing "index table has expected columns"
+ (is (= ["document" "document_hash" "gated_at" "id" "model" "model_id" "updated_at"]
+ (->> (jdbc/execute! (semantic.env/get-pgvector-datasource!)
+ (sql/format {:select [:column_name]
+ :from [:information_schema.columns]
+ :where [[:= :table_name [:inline "index_gate"]]]}))
+ (map :columns/column_name)
+ sort
+ vec))))))
(catch Throwable e
(log/fatal e)
(throw e))))
@@ -229,42 +229,151 @@
(deftest dynamic-schema-migration-test
(mt/with-premium-features #{:semantic-search}
(semantic.tu/with-test-db-defaults!
- (with-redefs [semantic.pgvector-api/index-documents! (constantly nil)]
- (semantic.core/init! (semantic.tu/mock-documents) nil)
- ;; add column to index table
- (let [original-dynamic-schema semantic.db.migration.impl/dynamic-schema-version]
- (with-redefs-fn {#'semantic.db.migration.impl/migrate-dynamic-schema!
- (fn [tx _opts]
- (let [table_names (->> (jdbc/execute! tx
- (sql/format {:select [:table_name]
- :from [:index_metadata]
- :where [[:< :index_version semantic.db.migration.impl/dynamic-schema-version]]
- :group-by [:table_name]}))
- (map :index_metadata/table_name)
- set)]
- (doseq [table_name table_names]
- (when-not (has-column?! tx table_name "new_col")
- (jdbc/execute! tx (sql/format {:alter-table [table_name] :add-column [[:new_col :int]]}))))
- (jdbc/execute! tx (sql/format {:update :index_metadata
- :set {:index_version semantic.db.migration.impl/dynamic-schema-version}
- :where [[:in :table_name table_names]]}))))
- #'semantic.db.migration.impl/dynamic-schema-version (inc original-dynamic-schema)}
- (fn []
- ;; Trigger migration by next initialization attempt
- (semantic.core/init! (semantic.tu/mock-documents) nil)
- (let [#:index_metadata{:keys [index_version table_name]}
- (jdbc/execute-one! (semantic.db.datasource/ensure-initialized-data-source!)
- (sql/format
- {:select [:*]
- :from [:index_metadata]}))]
- (testing "Index metadata table ids were updated"
- (is (= index_version semantic.db.migration.impl/dynamic-schema-version)))
- (testing "Index table contains new column"
- (is (has-column?! (semantic.db.datasource/ensure-initialized-data-source!)
- table_name "new_col")))))))))))
+ (semantic.core/init! (semantic.tu/mock-documents) nil)
+ ;; add column to index table
+ (let [original-dynamic-schema semantic.db.migration.impl/dynamic-schema-version]
+ (with-redefs-fn {#'semantic.db.migration.impl/migrate-dynamic-schema!
+ (fn [tx _opts]
+ (let [table_names (->> (jdbc/execute! tx
+ (sql/format {:select [:table_name]
+ :from [:index_metadata]
+ :where [[:< :index_version semantic.db.migration.impl/dynamic-schema-version]]
+ :group-by [:table_name]}))
+ (map :index_metadata/table_name)
+ set)]
+ (doseq [table_name table_names]
+ (when-not (has-column?! tx table_name "new_col")
+ (jdbc/execute! tx (sql/format {:alter-table [table_name] :add-column [[:new_col :int]]}))))
+ (jdbc/execute! tx (sql/format {:update :index_metadata
+ :set {:index_version semantic.db.migration.impl/dynamic-schema-version}
+ :where [[:in :table_name table_names]]}))))
+ #'semantic.db.migration.impl/dynamic-schema-version (inc original-dynamic-schema)}
+ (fn []
+ ;; Trigger migration by next initialization attempt
+ (semantic.core/init! (semantic.tu/mock-documents) nil)
+ (let [#:index_metadata{:keys [index_version table_name]}
+ (jdbc/execute-one! (semantic.db.datasource/ensure-initialized-data-source!)
+ (sql/format
+ {:select [:*]
+ :from [:index_metadata]}))]
+ (testing "Index metadata table ids were updated"
+ (is (= index_version semantic.db.migration.impl/dynamic-schema-version)))
+ (testing "Index table contains new column"
+ (is (has-column?! (semantic.db.datasource/ensure-initialized-data-source!)
+ table_name "new_col"))))))))))
(deftest schema-version-match-default-version-test
(testing "Default schema should match dynamic schema version"
(is (= semantic.db.migration.impl/dynamic-schema-version
(-> (semantic.index/default-index semantic.tu/mock-embedding-model)
:version)))))
+
+(defn- active-index-table-name!
+ [pgvector index-metadata]
+ (let [{:keys [control-table-name metadata-table-name]} index-metadata]
+ (->> (jdbc/execute-one! pgvector
+ (sql/format {:select [:im.table_name]
+ :from [[(keyword metadata-table-name) :im]]
+ :join [[(keyword control-table-name) :ic] [:= :ic.active_id :im.id]]}))
+ vals
+ first)))
+
+(defn- insert-index-row!
+ "Raw INSERT into the active index table for backfill tests. Fills in the minimum NOT NULL
+ columns with placeholder values; `row` supplies anything else (model, model_id,
+ collection_type, root_collection_type)."
+ [pgvector kw-tbl row]
+ (jdbc/execute! pgvector
+ (sql/format {:insert-into kw-tbl
+ :values [(merge {:name [:inline ""]
+ :content [:inline ""]
+ :text_search_vector [:to_tsvector [:inline "simple"] [:inline ""]]
+ :text_search_with_native_query_vector [:to_tsvector [:inline "simple"] [:inline ""]]
+ :embedding [:raw "'[0,0,0,0]'::vector"]}
+ row)]}
+ :quoted true)))
+
+(deftest migration-4-backfill-test
+ (testing "migration 4 backfills root_collection_type from the gate doc, then via an appdb walk of the Library forest"
+ (mt/with-premium-features #{:semantic-search}
+ ;; :mock-initialized puts the 4-dim mock embedding model in scope so the placeholder
+ ;; `[0,0,0,0]` embedding in [[insert-index-row!]] matches the index column's dimensions.
+ (semantic.tu/with-test-db! {:mode :mock-initialized}
+ ;; Stand up a Library tree: library root → library-data sub → library-data sub-sub,
+ ;; plus a library-metrics sub under the root. The backfill should resolve every nested
+ ;; collection's `root_collection_type` to the top-level Library's own `:type`.
+ (mt/with-temp [:model/Collection {lib-id :id} {:name "Library" :type collection/library-collection-type :location "/"}
+ :model/Collection {data-id :id} {:name "Data"
+ :type collection/library-data-collection-type
+ :location (str "/" lib-id "/")}
+ :model/Collection {nested-data-id :id} {:name "Nested Data"
+ :type collection/library-data-collection-type
+ :location (str "/" lib-id "/" data-id "/")}
+ :model/Collection {metrics-id :id} {:name "Metrics"
+ :type collection/library-metrics-collection-type
+ :location (str "/" lib-id "/")}
+ :model/Collection {other-id :id} {:name "Regular" :location "/"}]
+ (let [pgvector (semantic.env/get-pgvector-datasource!)
+ index-metadata (semantic.env/get-index-metadata)
+ gate-tbl (keyword (:gate-table-name index-metadata))
+ meta-tbl (keyword (:metadata-table-name index-metadata))
+ kw-tbl (keyword (active-index-table-name! pgvector index-metadata))]
+ ;; row 1: NULL root_collection_type; matching gate doc carries "library" in its document JSON
+ (insert-index-row! pgvector kw-tbl {:model [:inline "card"] :model_id [:inline "1"]})
+ ;; row 2: collection_id = top-level Library (root of its own tree)
+ (insert-index-row! pgvector kw-tbl {:model [:inline "card"]
+ :model_id [:inline "2"]
+ :collection_id lib-id})
+ ;; row 3: collection_id = first-level library-data sub-collection
+ (insert-index-row! pgvector kw-tbl {:model [:inline "card"]
+ :model_id [:inline "3"]
+ :collection_id data-id})
+ ;; row 4: collection_id = deeper library-data sub-sub-collection
+ (insert-index-row! pgvector kw-tbl {:model [:inline "card"]
+ :model_id [:inline "4"]
+ :collection_id nested-data-id})
+ ;; row 5: collection_id = library-metrics sub-collection — still resolves to "library"
+ (insert-index-row! pgvector kw-tbl {:model [:inline "card"]
+ :model_id [:inline "5"]
+ :collection_id metrics-id})
+ ;; row 6: collection_id = a non-library collection — backfill must leave NULL
+ (insert-index-row! pgvector kw-tbl {:model [:inline "card"]
+ :model_id [:inline "6"]
+ :collection_id other-id})
+ ;; row 7: already-populated root_collection_type — backfill must leave it alone
+ (insert-index-row! pgvector kw-tbl {:model [:inline "card"]
+ :model_id [:inline "7"]
+ :root_collection_type [:inline "library-data"]})
+ ;; Gate doc for row 1 — JSON includes root_collection_type
+ (jdbc/execute! pgvector
+ (sql/format {:insert-into gate-tbl
+ :values [{:id [:inline "card_1"]
+ :model [:inline "card"]
+ :model_id [:inline "1"]
+ :updated_at [:now]
+ :document [:cast [:inline "{\"root_collection_type\":\"library\"}"] :jsonb]
+ :document_hash [:inline "h"]}]}
+ :quoted true))
+ ;; Roll metadata.index_version back to 3 so migration 4 re-runs against the table.
+ (jdbc/execute! pgvector
+ (sql/format {:update meta-tbl
+ :set {:index_version 3}}))
+ ;; Trigger re-migration via init.
+ (semantic.core/init! (semantic.tu/mock-documents) nil)
+ ;; Verify each backfill branch.
+ (let [rows-by-id (->> (jdbc/execute! pgvector
+ (sql/format {:select [:model_id :root_collection_type]
+ :from [kw-tbl]
+ :order-by [:model_id]}
+ :quoted true)
+ {:builder-fn jdbc.rs/as-unqualified-maps})
+ (map (juxt :model_id :root_collection_type))
+ (into {}))]
+ (is (= {"1" "library" ; pulled from gate.document->>'root_collection_type'
+ "2" "library" ; collection_id IS the library root
+ "3" "library" ; library-data sub-collection — walks up to library
+ "4" "library" ; deeper library-data sub-sub — still walks up
+ "5" "library" ; library-metrics sub-collection — still walks up
+ "6" nil ; non-library collection, no gate doc
+ "7" "library-data"} ; pre-existing value preserved
+ rows-by-id)))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/scoring_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/scoring_test.clj
index df309a73149..13e39e2644a 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/scoring_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/scoring_test.clj
@@ -1,6 +1,7 @@
(ns metabase-enterprise.semantic-search.scoring-test
(:require
[clojure.core.memoize :as memoize]
+ [clojure.string :as str]
[clojure.test :refer :all]
[metabase-enterprise.semantic-search.index :as semantic.index]
[metabase-enterprise.semantic-search.scoring :as semantic.scoring]
@@ -235,6 +236,56 @@
["collection" 2 "collection official"]]
(search-results* "collection")))))))
+(deftest library-test
+ (mt/with-premium-features #{:semantic-search}
+ (testing "Items whose root ancestor collection is a library type rank above non-library items"
+ (with-index-contents!
+ [{:model "card" :id 1 :name "plain card"}
+ {:model "card" :id 2 :name "lib-tree card library" :root_collection_type "library"}
+ {:model "card" :id 3 :name "lib-tree card library-data" :root_collection_type "library-data"}
+ {:model "card" :id 4 :name "lib-tree card library-metrics" :root_collection_type "library-metrics"}
+ {:model "card" :id 5 :name "trashed card" :root_collection_type "trash"}]
+ (let [in-library? (fn [[_ _ nm]] (str/includes? nm "lib-tree"))]
+ (testing "with positive :library weight, library items come first"
+ (is (= [true true true false false]
+ (map in-library? (semantic.tu/with-weights {:library 1} (search-results* "card"))))))
+ (testing "with negative :library weight, library items come last"
+ (is (= [false false true true true]
+ (map in-library? (semantic.tu/with-weights {:library -1} (search-results* "card")))))))))))
+
+(deftest data-layer-test
+ (mt/with-premium-features #{:semantic-search}
+ (testing ":data-layer scorer reads the active per-tier weight via :data-layer/* params"
+ (with-index-contents!
+ [{:model "table" :id 1 :name "table no layer"}
+ {:model "table" :id 2 :name "table final" :data_layer "final"}
+ {:model "table" :id 3 :name "table internal" :data_layer "internal"}
+ {:model "table" :id 4 :name "table hidden" :data_layer "hidden"}]
+ (testing "final tables lead when only :data-layer/final is positive"
+ (is (= 2 (-> (semantic.tu/with-weights {:data-layer 1 :data-layer/final 1}
+ (search-results* "table"))
+ first second))))
+ (testing "internal tables lead when only :data-layer/internal is positive"
+ (is (= 3 (-> (semantic.tu/with-weights {:data-layer 1 :data-layer/internal 1}
+ (search-results* "table"))
+ first second))))
+ (testing "hidden tables lead when only :data-layer/hidden is positive"
+ (is (= 4 (-> (semantic.tu/with-weights {:data-layer 1 :data-layer/hidden 1}
+ (search-results* "table"))
+ first second))))))
+ (testing "tier ordering: final > internal > hidden under :metabot magnitudes"
+ (with-index-contents!
+ [{:model "table" :id 1 :name "foo table final" :data_layer "final"}
+ {:model "table" :id 2 :name "foo table internal" :data_layer "internal"}
+ {:model "table" :id 3 :name "foo table hidden" :data_layer "hidden"}]
+ (is (= [1 2 3]
+ (->> (semantic.tu/with-weights {:data-layer 33
+ :data-layer/final 1
+ :data-layer/internal 0.3
+ :data-layer/hidden 0.03}
+ (search-results* "foo table"))
+ (map second))))))))
+
(deftest verified-test
(with-index-contents!
[{:model "card" :id 1 :name "card normal" :verified false}
diff --git a/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj b/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj
index 81d47ec0188..0bda2362395 100644
--- a/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj
+++ b/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj
@@ -629,10 +629,8 @@
(-> %
(driver-api/assoc-field-options ::sql.qp/wrap-in-case true)
(driver-api/assoc-field-options ::sql.qp/add-cast :bit))
-
(sql.qp.boolean-to-comparison/boolean-expression-clause? %)
(driver-api/assoc-field-options % ::sql.qp/add-cast :bit)
-
:else
%)]
(->> (update query :fields #(mapv maybe-add-cast %))
diff --git a/src/metabase/lib/schema/expression.cljc b/src/metabase/lib/schema/expression.cljc
index 7331731553b..f4e8611b545 100644
--- a/src/metabase/lib/schema/expression.cljc
+++ b/src/metabase/lib/schema/expression.cljc
@@ -266,12 +266,9 @@
#(cond
(vector? %) ;; non-literal (checked above)
true
-
(not (int? %))
false
-
(not (pos? %))
false
-
:else
true)]])
diff --git a/src/metabase/search/appdb/scoring.clj b/src/metabase/search/appdb/scoring.clj
index e053e956575..b8a289b907b 100644
--- a/src/metabase/search/appdb/scoring.clj
+++ b/src/metabase/search/appdb/scoring.clj
@@ -2,7 +2,6 @@
(:require
[clojure.core.memoize :as memoize]
[honey.sql.helpers :as sql.helpers]
- [metabase.collections.models.collection :as collection]
[metabase.config.core :as config]
[metabase.premium-features.core :refer [defenterprise]]
[metabase.search.appdb.index :as search.index]
@@ -64,26 +63,8 @@
;; in this case, we need to transform the string into a pattern in code, so forced to use helper
(search.scoring/prefix [:lower :search_index.name] (u/lower-case-en search-string))
[:inline 0])
- ;; :library matches items whose root (top-level) ancestor collection is one of the library types.
- ;; The :root-collection-type attr is computed at ingestion time by walking the collection's
- ;; materialized path (see metabase.collections.models.collection/root-collection-type), so items
- ;; in arbitrarily deep sub-collections of a library tree still match here.
- :library [:case
- (into [:or]
- (for [t (sort collection/library-collection-types)]
- [:= :search_index.root_collection_type [:inline t]]))
- [:inline 1]
- :else [:inline 0]]
- ;; :data-layer mirrors the :model/* pattern — one scorer, per-tier weights live under :data-layer/*
- ;; (see metabase.search.config/scorer-param). Final/internal/hidden are mutually exclusive.
- :data-layer [:case
- [:= :search_index.data_layer [:inline "final"]]
- [:inline (or (search.config/scorer-param search-ctx :data-layer :final) 0)]
- [:= :search_index.data_layer [:inline "internal"]]
- [:inline (or (search.config/scorer-param search-ctx :data-layer :internal) 0)]
- [:= :search_index.data_layer [:inline "hidden"]]
- [:inline (or (search.config/scorer-param search-ctx :data-layer :hidden) 0)]
- :else [:inline 0]]}))
+ :library (search.scoring/library-score-expr)
+ :data-layer (search.scoring/data-layer-score-expr search-ctx)}))
(defenterprise scorers
"Return the select-item expressions used to calculate the score for each search result."
diff --git a/src/metabase/search/config.clj b/src/metabase/search/config.clj
index c1316ce186c..d2f02cb054f 100644
--- a/src/metabase/search/config.clj
+++ b/src/metabase/search/config.clj
@@ -84,13 +84,18 @@
:user-recency 5
:dashboard 0
:model 2
- :official-collection 1
- :verified 1
:view-count 2
:text 5
:mine 1
:exact 5
:prefix 0
+ ;; User-curated tiers, from strongest to weakest.
+ ;; :library at 200 exceeds :official-collection + :verified (80 each) combined,
+ ;; so a library item always outranks a non-library one.
+ ;; Base text/recency scorers (0–5 range) only break ties *within* a tier.
+ :library 200
+ :official-collection 80
+ :verified 80
;; RRF is the "Reciprocal Rank Fusion" score used by the semantic search backend to blend semantic and keyword scores
:rrf 500}
:command-palette
@@ -108,14 +113,28 @@
:model/dataset 1
:model/metric 1
:model/question 0}
+ ;; TODO: lift :data-layer up to :default. It's a structural signal (every visible warehouse
+ ;; table gets `data_layer "final"`), so the +33 boost flips orderings across every search
+ ;; surface and breaks e2e specs that pin specific top results. To make progress:
+ ;;
+ ;; - Update the e2e snapshot in `e2e/snapshot-creators/default.cy.snap.js` so the sample-DB
+ ;; tables aren't all at `final` (mark non-essential ones `internal`/`hidden`), or
+ ;; - Rewrite the brittle assertions to not depend on a specific top result.
+ ;;
+ ;; Affected specs (broken by an earlier lift attempt):
+ ;;
+ ;; - `search.cy.spec.js`
+ ;; - `models.cy.spec.js`
+ ;; - `documents.cy.spec.ts`
+ ;; - `document-links.cy.spec.ts`
+ ;; - `custom-viz.cy.spec.ts`
+ ;; - `search-snowplow.cy.spec.js`
:metabot
- {:library 100
- :official-collection 80
- :verified 80
- :data-layer 1 ; overall multiplier; per-tier weights live under :data-layer/* below
- :data-layer/final 33
- :data-layer/internal 10
- :data-layer/hidden 1}})
+ {:data-layer 33
+ :data-layer/final 1 ; ≈ 33
+ :data-layer/internal 0.3 ; ≈ 10
+ :data-layer/hidden 0.03 ; ≈ 1
+ }})
(def ^:private FilterDef
"A relaxed definition, capturing how we can write the filter - with some fields omitted."
diff --git a/src/metabase/search/scoring.clj b/src/metabase/search/scoring.clj
index 5ecc47a4348..8d8f1a523f6 100644
--- a/src/metabase/search/scoring.clj
+++ b/src/metabase/search/scoring.clj
@@ -3,6 +3,7 @@
[clojure.string :as str]
[honey.sql.helpers :as sql.helpers]
[metabase.app-db.core :as mdb]
+ [metabase.collections.models.collection :as collection]
[metabase.search.config :as search.config]
[metabase.util.honey-sql-2 :as h2x]))
@@ -84,6 +85,29 @@
[:= :search_index.model [:inline "metric"]] [:inline "card"]
:else :search_index.model]]]}))
+(defn library-score-expr
+ "Score expression: 1 when `:root_collection_type` is one of the library collection types, else 0."
+ []
+ ;; `:root-collection-type` is set at ingestion via `collection/root-collection-type` (walks the
+ ;; materialized path), so deeply nested items in a library tree still match.
+ [:case
+ (into [:or]
+ (for [t (sort collection/library-collection-types)]
+ [:= :root_collection_type [:inline t]]))
+ [:inline 1]
+ :else [:inline 0]])
+
+(defn data-layer-score-expr
+ "Score expression: per-tier weight when `:data_layer` is `final`/`internal`/`hidden`, else 0.
+ Per-tier weights are read from `:data-layer/*` keys via [[search.config/scorer-param]]."
+ [search-ctx]
+ (let [tier-weight #(or (search.config/scorer-param search-ctx :data-layer %) 0)]
+ [:case
+ [:= :data_layer [:inline "final"]] [:inline (tier-weight :final)]
+ [:= :data_layer [:inline "internal"]] [:inline (tier-weight :internal)]
+ [:= :data_layer [:inline "hidden"]] [:inline (tier-weight :hidden)]
+ :else [:inline 0]]))
+
(defn model-rank-expr
"Score an item based on its :model type."
[search-ctx]
diff --git a/test/metabase/search/appdb/scoring_test.clj b/test/metabase/search/appdb/scoring_test.clj
index fb0f50bbe9e..cff8ab9c3dc 100644
--- a/test/metabase/search/appdb/scoring_test.clj
+++ b/test/metabase/search/appdb/scoring_test.clj
@@ -2,6 +2,7 @@
(:require
[clojure.core.memoize :as memoize]
[clojure.set :as set]
+ [clojure.string :as str]
[clojure.test :refer :all]
[metabase.app-db.core :as mdb]
[metabase.search.appdb.index :as search.index]
@@ -326,15 +327,14 @@
:model/Collection sub-sub {:name "sub of sub" :location (format "/%d/%d/" (:id lib) (:id sub))}
:model/Collection other {:name "non-library" :location "/"}]
(with-index-contents
- [{:model "card" :id 1 :name "card plain" :collection_id (:id other) :collection_location (:location other)}
- {:model "card" :id 2 :name "card in library" :collection_id (:id lib) :collection_location (:location lib) :collection_type "library"}
- {:model "card" :id 3 :name "card in library-data" :collection_id (:id lib-data) :collection_location (:location lib-data) :collection_type "library-data"}
- {:model "card" :id 4 :name "card in lib-metrics" :collection_id (:id lib-met) :collection_location (:location lib-met) :collection_type "library-metrics"}
- {:model "card" :id 5 :name "card in sub of lib" :collection_id (:id sub) :collection_location (:location sub)}
- {:model "card" :id 6 :name "card in sub of sub" :collection_id (:id sub-sub) :collection_location (:location sub-sub)}
- {:model "card" :id 7 :name "card in trash" :collection_type "trash"}]
- (let [library-id? #{2 3 4 5 6}
- in-library? (fn [[_ id _]] (boolean (library-id? id)))]
+ [{:model "card" :id 1 :name "plain card" :collection_id (:id other) :collection_location (:location other)}
+ {:model "card" :id 2 :name "lib-tree card library" :collection_id (:id lib) :collection_location (:location lib) :collection_type "library"}
+ {:model "card" :id 3 :name "lib-tree card library-data" :collection_id (:id lib-data) :collection_location (:location lib-data) :collection_type "library-data"}
+ {:model "card" :id 4 :name "lib-tree card library-metrics" :collection_id (:id lib-met) :collection_location (:location lib-met) :collection_type "library-metrics"}
+ {:model "card" :id 5 :name "lib-tree card sub" :collection_id (:id sub) :collection_location (:location sub)}
+ {:model "card" :id 6 :name "lib-tree card sub-sub" :collection_id (:id sub-sub) :collection_location (:location sub-sub)}
+ {:model "card" :id 7 :name "trashed card" :collection_type "trash"}]
+ (let [in-library? (fn [[_ _ nm]] (str/includes? nm "lib-tree"))]
(testing "with positive :library weight, items inside library trees come first"
(is (= [true true true true true false false]
(map in-library? (with-weights {:library 1} (search-results* "card"))))))
@@ -361,15 +361,15 @@
(is (= 4 (-> (with-weights {:data-layer 1 :data-layer/hidden 1}
(search-results* "table"))
first second))))))
- (testing "Metabot tier ordering: final > internal > hidden when all weights active"
+ (testing "tier ordering: final > internal > hidden under :metabot magnitudes"
(with-index-contents
- [{:model "table" :id 1 :name "metabot table final" :data_layer "final"}
- {:model "table" :id 2 :name "metabot table internal" :data_layer "internal"}
- {:model "table" :id 3 :name "metabot table hidden" :data_layer "hidden"}]
+ [{:model "table" :id 1 :name "foo table final" :data_layer "final"}
+ {:model "table" :id 2 :name "foo table internal" :data_layer "internal"}
+ {:model "table" :id 3 :name "foo table hidden" :data_layer "hidden"}]
(is (= [1 2 3]
- (->> (with-weights {:data-layer 1
- :data-layer/final 33
- :data-layer/internal 10
- :data-layer/hidden 1}
- (search-results* "metabot table"))
+ (->> (with-weights {:data-layer 33
+ :data-layer/final 1
+ :data-layer/internal 0.3
+ :data-layer/hidden 0.03}
+ (search-results* "foo table"))
(map second)))))))
diff --git a/test/metabase/search/config_test.clj b/test/metabase/search/config_test.clj
index c2ce587c9d3..a258ee06bba 100644
--- a/test/metabase/search/config_test.clj
+++ b/test/metabase/search/config_test.clj
@@ -22,17 +22,3 @@
(search.config/filter-default :search.engine/in-place :command-palette :filter-items-in-personal-collection)))
(is (= "exclude-others"
(search.config/filter-default :search.engine/in-place :search-app :filter-items-in-personal-collection)))))
-
-(deftest metabot-weights-test
- (testing "the :metabot context defines the tuned curation tier weights"
- (is (=? {:library 100
- :official-collection 80
- :verified 80
- :data-layer 1
- :data-layer/final 33
- :data-layer/internal 10
- :data-layer/hidden 1}
- (search.config/weights {:context :metabot}))))
- (testing "request-level :weights override beats :metabot static weights"
- (is (= 7 (-> (search.config/weights {:context :metabot :weights {:library 7}})
- :library)))))