Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Semantic search speedup (#70317)

Oleksandr Solovyovc8b1767e6635
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration/impl.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration/impl.clj
index 63e6f56cbd0..3b17db7f9fb 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration/impl.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration/impl.clj
@@ -37,11 +37,29 @@
(def dynamic-schema-version
"Code version of dynamic schema (index_table_xyzs). If higher than what's found in db dynamic schema migration will
be attempted."
- 1)
+ 2)
+
+(defn- add-personal-owner-id-column!
+ "Migration 2: Add `personal_owner_id` column to index tables for SQL-level personal collection filtering."
+ [tx index-metadata]
+ (let [index-metadata (keyword (:metadata-table-name index-metadata))
+ execute! (fn [q] (jdbc/execute! tx (sql/format q)))
+ table-names (->> (execute! {:select-distinct [:table_name]
+ :from [index-metadata]
+ :where [[:< :index_version 2]]})
+ (mapcat vals))]
+ (when (seq table-names)
+ (doseq [table-name table-names]
+ (execute! {:alter-table [(keyword table-name)]
+ :add-column [[:personal_owner_id :int :if-not-exists]]}))
+ (execute! {:update index-metadata
+ :set {:index_version 2}
+ :where [[:in :table_name (vec table-names)]]}))))
(defn migrate-dynamic-schema!
"Migrate runtime-managed schema, ie. schema of `index_table_...` tables. Migration author is responsible for removing
leftovers if necessary."
- [_tx {_index-metadata :index-metadata _embedding-model :embedding-model :as _opts}]
+ [tx {:keys [index-metadata] :as _opts}]
;; migration 1: all tables dropped in schema migration in single function call
- )
+ ;; migration 2: add personal_owner_id column to index tables
+ (add-personal-owner-id-column! tx index-metadata))
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj
index 9004220c1c7..94cf79185fe 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj
@@ -2,6 +2,7 @@
(:require
[buddy.core.codecs :as buddy-codecs]
[buddy.core.hash :as buddy-hash]
+ [clojure.set :as set]
[clojure.string :as str]
[com.climate.claypoole :as cp]
[honey.sql :as sql]
@@ -53,6 +54,7 @@
[:model :text :not-null]
[:model_id :text :not-null]
[:collection_id :int]
+ [:personal_owner_id :int]
[:creator_id :int]
[:database_id :int]
[:last_editor_id :int]
@@ -115,14 +117,43 @@
(= 1 b) true
:else (throw (ex-info "Unexpected boolean value" {:v b}))))
+(defn- batch-resolve-personal-owner-ids
+ "Given a seq of collection-ids, return a map of collection-id -> personal_owner_id.
+ Collections not in any personal tree will be absent from the map.
+ Uses at most 2 queries regardless of the number of collection-ids."
+ [collection-ids]
+ (when-let [collection-ids (not-empty (set (remove nil? collection-ids)))]
+ (let [colls (t2/select [:model/Collection :id :personal_owner_id :location]
+ :id [:in collection-ids])
+ {personal true
+ non-personal false} (group-by (comp some? :personal_owner_id) colls)
+ direct (into {} (map (juxt :id :personal_owner_id)) personal)
+ roots (keep (fn [{:keys [id location]}]
+ (when-let [root-id (some-> location (str/split #"/" 3) second parse-long)]
+ [id root-id]))
+ non-personal)
+ unknown-roots (set/difference (set (map second roots)) (set (keys direct)))
+ root-id->owner (into direct
+ (map (juxt :id :personal_owner_id))
+ (when (seq unknown-roots)
+ (t2/select [:model/Collection :id :personal_owner_id]
+ :id [:in unknown-roots]
+ :personal_owner_id [:not= nil])))]
+ (into direct
+ (keep (fn [[coll-id root-id]]
+ (when-let [owner (get root-id->owner root-id)]
+ [coll-id owner])))
+ roots))))
+
(defn- doc->db-record
"Convert a document to a database record with a provided embedding."
- [embedding-vec {:keys [model id searchable_text embeddable_text native_query created_at creator_id updated_at
- last_editor_id archived verified official_collection database_id collection_id display_type legacy_input
- pinned dashboardcard_count view_count last_viewed_at] :as doc}]
+ [owner-ids {:keys [model id embedding searchable_text embeddable_text native_query created_at creator_id updated_at
+ last_editor_id archived verified official_collection database_id collection_id display_type legacy_input
+ pinned dashboardcard_count view_count last_viewed_at] :as doc}]
{:model model
:model_id id
:collection_id collection_id
+ :personal_owner_id (get owner-ids collection_id)
:creator_id creator_id
:database_id database_id
:last_editor_id last_editor_id
@@ -139,8 +170,8 @@
:model_updated_at (some-> updated_at to-instant)
:last_viewed_at (some-> last_viewed_at to-instant)
:legacy_input [:cast (json/encode legacy_input) :jsonb]
- :metadata [:cast (json/encode doc) :jsonb]
- :embedding [:raw (format-embedding embedding-vec)]
+ :metadata [:cast (json/encode (dissoc doc :embedding)) :jsonb]
+ :embedding [:raw (format-embedding embedding)]
:text_search_vector (if (:name doc)
[:||
(search/weighted-tsvector "A" (:name doc))
@@ -177,14 +208,14 @@
(log/warn e "Failed to set :metabase-search/semantic-index-size metric"))))
(defn- batch-update!
- [connectable table-name records->sql documents embeddings]
+ [connectable table-name records->sql documents]
(when (seq documents)
(u/prog1 (->> documents (map :model) frequencies)
(u/profile (str "Semantic index database update of " (count documents) " documents " <>)
- (doseq [batch (->> (map vector documents embeddings)
- (map (fn [[doc embedding]] (doc->db-record embedding doc)))
- (partition-all *batch-size*))]
- (jdbc/execute! connectable (records->sql batch)))
+ (let [owner-ids (batch-resolve-personal-owner-ids (map :collection_id documents))]
+ (doseq [batch (->> (map #(doc->db-record owner-ids %) documents)
+ (partition-all *batch-size*))]
+ (jdbc/execute! connectable (records->sql batch))))
(analytics-set-index-size! connectable table-name)))))
(defn- execute-with-counts [connectable model ids sql]
@@ -247,7 +278,7 @@
(let [table-name (or table-name (model-table-name embedding-model))]
{:embedding-model embedding-model
:table-name table-name
- :version 1}))
+ :version 2}))
(defn- upsert-embedding!-fn [connectable index text->docs]
(fn [text->embedding]
@@ -269,8 +300,7 @@
(sql.helpers/on-conflict :model :model_id)
(sql.helpers/do-update-set (db-records->update-set db-records))
sql-format-quoted))
- batch-documents
- (map :embedding batch-documents)))))
+ batch-documents))))
(defn- unwrap-pgobject
[^PGobject obj]
@@ -469,11 +499,42 @@
(create-index-table-if-not-exists! db index)
(jdbc/execute! db ["select table_name from INFORMATION_SCHEMA.tables where table_name like 'index_table_%'"]))
+(defn- personal-collection-filter
+ "Generate a WHERE condition for personal collection filtering based on the filter type.
+
+ | Filter | Personal | Others' Personal | Shared Coll. | No Coll. |
+ |----------------|----------|------------------|--------------|----------|
+ | all | ✅ | ✅ | ✅ | ✅ |
+ | only-mine | ✅ | ❌ | ❌ | ❌ |
+ | only | ✅ | ✅ | ❌ | ❌ |
+ | exclude | ❌ | ❌ | ✅ | ✅ |
+ | exclude-others | ✅ | ❌ | ✅ | ✅ |"
+ [{:keys [filter-items-in-personal-collection current-user-id]}]
+ (case (or filter-items-in-personal-collection "all")
+ "all"
+ nil
+
+ "only-mine"
+ [:= :personal_owner_id current-user-id]
+
+ "only"
+ [:is-not :personal_owner_id nil]
+
+ "exclude"
+ [:is :personal_owner_id nil]
+
+ "exclude-others"
+ [:or
+ [:is :personal_owner_id nil]
+ [:= :personal_owner_id current-user-id]]))
+
(defn- search-filters
"Generate WHERE conditions based on search context filters."
- [{:keys [archived? verified models created-at created-by last-edited-at last-edited-by table-db-id ids display-type]}]
+ [{:keys [archived? verified models created-at created-by last-edited-at last-edited-by
+ table-db-id ids display-type] :as search-context}]
(let [conditions (filter some?
- [(when (some? archived?)
+ [(personal-collection-filter search-context)
+ (when (some? archived?)
[:= :archived archived?])
(when (some? verified)
[:= :verified verified])
@@ -505,6 +566,7 @@
[:model :model]
[:model_id :model_id]
[:collection_id :collection_id]
+ [:personal_owner_id :personal_owner_id]
[:creator_id :creator_id]
[:database_id :database_id]
[:last_editor_id :last_editor_id]
@@ -693,90 +755,6 @@
result))
-(defn- get-personal-collection-ids
- "Get the set of personal collection IDs by extracting root collection IDs from locations."
- [collections-map]
- (let [root-collection-ids (->> collections-map
- vals
- (map :location)
- (keep (fn [location]
- (when-let [match (re-find #"^/(\d+)/" location)]
- (parse-long (second match)))))
- distinct)]
- (when (seq root-collection-ids)
- (->> (t2/select [:collection :id]
- :id [:in root-collection-ids]
- :personal_owner_id [:not= nil])
- (map :id)
- set))))
-
-(defn- filter-by-collection
- "Filter documents based on personal collection preferences.
- Equivalent to metabase.search.filter/personal-collections-where-clause but operates on docs in memory.
-
- | Filter | Personal | Others' Personal | Shared Coll. | No Coll. |
- |----------------|----------|------------------|--------------|----------|
- | all | ✅ | ✅ | ✅ | ✅ |
- | only-mine | ✅ | ❌ | ❌ | ❌ |
- | only | ✅ | ✅ | ❌ | ❌ |
- | exclude | ❌ | ❌ | ✅ | ✅ |
- | exclude-others | ✅ | ❌ | ✅ | ✅ |
- "
- [docs {:keys [filter-items-in-personal-collection current-user-id]}]
- (let [filter-type (or filter-items-in-personal-collection "all")]
- (if (= filter-type "all")
- docs
- (let [collection-ids (keep :collection_id docs)
- collections-map (when (seq collection-ids)
- (->> (t2/select [:collection :id :location :personal_owner_id]
- :id [:in collection-ids])
- (into {} (map (juxt :id identity)))))
- personal-collection-ids (when (not= filter-type "only-mine")
- (get-personal-collection-ids collections-map))
- user-personal-collection-id (t2/select-one-pk :model/Collection :personal_owner_id [:= current-user-id])
- is-personal-collection? (fn [coll] (some? (:personal_owner_id coll)))
- is-owned-by-user? (fn [coll] (= (:personal_owner_id coll) current-user-id))
- is-in-user-personal-tree? (fn [coll] (str/starts-with? (:location coll) (str "/" user-personal-collection-id "/")))
- is-in-any-personal-tree? (fn [coll]
- (when-let [match (re-find #"^/(\d+)/" (:location coll))]
- (contains? personal-collection-ids (parse-long (second match)))))]
- (case filter-type
- "only-mine"
- (filterv (fn [doc]
- (when-let [collection (get collections-map (:collection_id doc))]
- (or (is-owned-by-user? collection)
- (is-in-user-personal-tree? collection))))
- docs)
-
- "only"
- (filterv (fn [doc]
- (when-let [collection (get collections-map (:collection_id doc))]
- (or (is-personal-collection? collection)
- (is-in-any-personal-tree? collection))))
- docs)
-
- "exclude"
- (filterv (fn [doc]
- (let [collection-id (:collection_id doc)]
- (or (nil? collection-id)
- (when-let [collection (get collections-map collection-id)]
- (and (not (is-personal-collection? collection))
- (not (is-in-any-personal-tree? collection)))))))
- docs)
-
- "exclude-others"
- (filterv (fn [doc]
- (let [collection-id (:collection_id doc)]
- (or (nil? collection-id)
- (when-let [collection (get collections-map collection-id)]
- (or (is-owned-by-user? collection)
- (is-in-user-personal-tree? collection)
- (and (not (is-personal-collection? collection))
- (not (is-in-any-personal-tree? collection))))))))
- docs)
-
- docs)))))
-
(defn- filter-by-collection-id
"Filter documents by collection and all descendant collections."
[docs collection-id]
@@ -793,23 +771,6 @@
(str/starts-with? (:location collection) (str "/" collection-id "/")))))))
docs)))
-(defn- apply-collection-filter
- "Apply personal collection filtering with logging."
- [search-context docs]
- (let [filter-type (:filter-items-in-personal-collection search-context)]
- (if (or (nil? filter-type) (= filter-type "all"))
- docs
- (let [timer (u/start-timer)
- filtered-docs (filter-by-collection docs search-context)
- time-ms (u/since-ms timer)]
- (log/debug "Collection filter" {:filter filter-type
- :before (count docs)
- :after (count filtered-docs)
- :dropped (- (count docs) (count filtered-docs))
- :time_ms time-ms})
- (analytics/inc! :metabase-search/semantic-collection-filter-ms time-ms)
- filtered-docs))))
-
(defn- apply-collection-id-filter
"Apply collection ID filtering with logging."
[search-context docs]
@@ -858,7 +819,6 @@
filter-timer (u/start-timer)
filtered-results (->> raw-results
filter-read-permitted
- (apply-collection-filter search-context)
(apply-collection-id-filter search-context)
(mapv search/collapse-id))
filter-time-ms (u/since-ms filter-timer)
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj
index 3e25c41ffb3..09b1e7546bc 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj
@@ -371,7 +371,6 @@
(is (contains? metric-names :metabase-search/semantic-embedding-ms))
(is (contains? metric-names :metabase-search/semantic-db-query-ms))
(is (contains? metric-names :metabase-search/semantic-permission-filter-ms))
- (is (contains? metric-names :metabase-search/semantic-collection-filter-ms))
(is (contains? metric-names :metabase-search/semantic-appdb-scores-ms)))
(testing "timing values are reasonable"
@@ -380,10 +379,8 @@
:metabase-search/semantic-embedding-ms
:metabase-search/semantic-db-query-ms
:metabase-search/semantic-permission-filter-ms
- :metabase-search/semantic-collection-filter-ms
:metabase-search/semantic-appdb-scores-ms} metric)]
(let [time-ms (if (#{:metabase-search/semantic-permission-filter-ms
- :metabase-search/semantic-collection-filter-ms
:metabase-search/semantic-appdb-scores-ms}
metric)
(first args)
@@ -400,10 +397,43 @@
(is (map? labels) "Should have labels map")
(is (contains? labels :embedding-model) "Should include embedding-model label")))))))))))
-(deftest filter-by-collection-test
- (testing "filter-by-collection function filters documents based on personal collection preferences"
- (let [user1-id (mt/user->id :rasta)
- user2-id (mt/user->id :crowberto)
+(deftest personal-collection-filter-test
+ (testing "personal-collection-filter generates correct SQL WHERE clauses"
+ (let [user-id 42]
+ (testing "filter-type 'all' returns nil (no filter)"
+ (is (nil? (#'semantic.index/personal-collection-filter
+ {:filter-items-in-personal-collection "all" :current-user-id user-id}))))
+
+ (testing "filter-type nil defaults to 'all' behavior"
+ (is (nil? (#'semantic.index/personal-collection-filter
+ {:filter-items-in-personal-collection nil :current-user-id user-id}))))
+
+ (testing "filter-type 'only-mine' returns only current user's personal collection items"
+ (is (= [:= :personal_owner_id user-id]
+ (#'semantic.index/personal-collection-filter
+ {:filter-items-in-personal-collection "only-mine" :current-user-id user-id}))))
+
+ (testing "filter-type 'only' returns all personal collection items (any user)"
+ (is (= [:is-not :personal_owner_id nil]
+ (#'semantic.index/personal-collection-filter
+ {:filter-items-in-personal-collection "only" :current-user-id user-id}))))
+
+ (testing "filter-type 'exclude' returns only shared and uncollected items"
+ (is (= [:is :personal_owner_id nil]
+ (#'semantic.index/personal-collection-filter
+ {:filter-items-in-personal-collection "exclude" :current-user-id user-id}))))
+
+ (testing "filter-type 'exclude-others' returns user's personal items plus shared/uncollected items"
+ (is (= [:or
+ [:is :personal_owner_id nil]
+ [:= :personal_owner_id user-id]]
+ (#'semantic.index/personal-collection-filter
+ {:filter-items-in-personal-collection "exclude-others" :current-user-id user-id})))))))
+
+(deftest batch-resolve-personal-owner-ids-test
+ (testing "batch-resolve-personal-owner-ids correctly resolves personal collection ownership"
+ (let [user1-id (mt/user->id :rasta)
+ user2-id (mt/user->id :crowberto)
user1-personal-coll-id (u/the-id (collection/user->personal-collection user1-id))
user2-personal-coll-id (u/the-id (collection/user->personal-collection user2-id))]
(mt/with-temp
@@ -411,48 +441,30 @@
:model/Collection {user1-sub-coll-id :id} {:location (str "/" user1-personal-coll-id "/") :name "User1 Sub"}
:model/Collection {user2-sub-coll-id :id} {:location (str "/" user2-personal-coll-id "/") :name "User2 Sub"}]
- (let [docs [{:id "doc1" :model "card" :collection_id user1-personal-coll-id}
- {:id "doc2" :model "card" :collection_id user2-personal-coll-id}
- {:id "doc3" :model "card" :collection_id shared-coll-id}
- {:id "doc4" :model "card" :collection_id nil}
- {:id "doc5" :model "card" :collection_id user1-sub-coll-id}
- {:id "doc6" :model "card" :collection_id user2-sub-coll-id}]]
-
- (testing "filter-type 'all' returns all documents unchanged"
- (let [context {:filter-items-in-personal-collection "all" :current-user-id user1-id}
- result (#'semantic.index/filter-by-collection docs context)]
- (is (= 6 (count result)))
- (is (= (set (map :id docs)) (set (map :id result))))))
-
- (testing "filter-type nil defaults to 'all' behavior"
- (let [context {:filter-items-in-personal-collection nil :current-user-id user1-id}
- result (#'semantic.index/filter-by-collection docs context)]
- (is (= 6 (count result)))
- (is (= (set (map :id docs)) (set (map :id result))))))
-
- (testing "filter-type 'only-mine' returns only current user's personal collection items"
- (let [context {:filter-items-in-personal-collection "only-mine" :current-user-id user1-id}
- result (#'semantic.index/filter-by-collection docs context)
- result-ids (set (map :id result))]
- (is (= #{"doc1" "doc5"} result-ids))))
-
- (testing "filter-type 'only' returns all personal collection items (any user)"
- (let [context {:filter-items-in-personal-collection "only" :current-user-id user1-id}
- result (#'semantic.index/filter-by-collection docs context)
- result-ids (set (map :id result))]
- (is (= #{"doc1" "doc2" "doc5" "doc6"} result-ids))))
-
- (testing "filter-type 'exclude' returns only shared and uncollected items"
- (let [context {:filter-items-in-personal-collection "exclude" :current-user-id user1-id}
- result (#'semantic.index/filter-by-collection docs context)
- result-ids (set (map :id result))]
- (is (= #{"doc3" "doc4"} result-ids))))
-
- (testing "filter-type 'exclude-others' returns user's personal items plus shared/uncollected items"
- (let [context {:filter-items-in-personal-collection "exclude-others" :current-user-id user1-id}
- result (#'semantic.index/filter-by-collection docs context)
- result-ids (set (map :id result))]
- (is (= #{"doc1" "doc3" "doc4" "doc5"} result-ids)))))))))
+ (testing "empty input returns empty map"
+ (is (empty? (#'semantic.index/batch-resolve-personal-owner-ids [])))
+ (is (empty? (#'semantic.index/batch-resolve-personal-owner-ids [nil]))))
+
+ (testing "shared collection is absent from result"
+ (is (empty? (#'semantic.index/batch-resolve-personal-owner-ids [shared-coll-id]))))
+
+ (testing "personal collections map to their owners"
+ (is (= {user1-personal-coll-id user1-id
+ user2-personal-coll-id user2-id}
+ (#'semantic.index/batch-resolve-personal-owner-ids
+ [user1-personal-coll-id user2-personal-coll-id]))))
+
+ (testing "sub-collections of personal collections map to root personal owner"
+ (is (= {user1-sub-coll-id user1-id
+ user2-sub-coll-id user2-id}
+ (#'semantic.index/batch-resolve-personal-owner-ids
+ [user1-sub-coll-id user2-sub-coll-id]))))
+
+ (testing "mixed input resolves all in one call"
+ (is (= {user1-personal-coll-id user1-id
+ user2-sub-coll-id user2-id}
+ (#'semantic.index/batch-resolve-personal-owner-ids
+ [user1-personal-coll-id user2-sub-coll-id shared-coll-id nil]))))))))
(deftest filter-can-read-indexed-entity-test
(mt/with-premium-features #{:semantic-search}
@@ -540,7 +552,8 @@
:id "123"
:searchable_text "test content"
:embeddable_text "test content"
- :creator_id 1}]
+ :creator_id 1
+ :embedding embedding-vec}]
(testing "MySQL-style integer booleans are converted to real booleans"
(let [doc-with-mysql-booleans (assoc base-doc
@@ -548,7 +561,7 @@
:official_collection 1
:pinned 0
:verified 1)
- result (#'semantic.index/doc->db-record embedding-vec doc-with-mysql-booleans)]
+ result (#'semantic.index/doc->db-record nil doc-with-mysql-booleans)]
(is (false? (:archived result)))
(is (true? (:official_collection result)))
(is (false? (:pinned result)))
@@ -560,7 +573,7 @@
:official_collection false
:pinned true
:verified false)
- result (#'semantic.index/doc->db-record embedding-vec doc-with-real-booleans)]
+ result (#'semantic.index/doc->db-record nil doc-with-real-booleans)]
(is (true? (:archived result)))
(is (false? (:official_collection result)))
(is (true? (:pinned result)))
@@ -568,7 +581,7 @@
(testing "nil boolean fields are handled correctly"
(let [doc-with-nil-booleans base-doc
- result (#'semantic.index/doc->db-record embedding-vec doc-with-nil-booleans)]
+ result (#'semantic.index/doc->db-record nil doc-with-nil-booleans)]
(is (nil? (:archived result)))
(is (nil? (:official_collection result)))
(is (nil? (:pinned result)))
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/migration_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/migration_test.clj
index abba02dfedc..e7d98456725 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/migration_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/migration_test.clj
@@ -28,8 +28,9 @@
(semantic.tu/with-test-db-defaults!
(letfn [(migrate-and-get-db-version
[attempted-version]
- (with-redefs [semantic.db.migration.impl/schema-version attempted-version
- semantic.db.migration.impl/migrate-schema! (constantly nil)]
+ (with-redefs [semantic.db.migration.impl/schema-version attempted-version
+ semantic.db.migration.impl/migrate-schema! (constantly nil)
+ semantic.db.migration.impl/migrate-dynamic-schema! (constantly nil)]
(log.capture/with-log-messages-for-level [messages [metabase-enterprise.semantic-search.db.migration :info]]
(semantic.db.connection/with-migrate-tx [tx]
(semantic.db.migration/maybe-migrate! tx nil)
@@ -190,6 +191,7 @@
"model_updated_at"
"name"
"official_collection"
+ "personal_owner_id"
"pinned"
"text_search_vector"
"text_search_with_native_query_vector"