The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Semantic search speedup (#70317)
Oleksandr Solovyovc8b1767e6635
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration/impl.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration/impl.cljindex 63e6f56cbd0..3b17db7f9fb 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration/impl.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/db/migration/impl.clj@@ -37,11 +37,29 @@(def dynamic-schema-version"Code version of dynamic schema (index_table_xyzs). If higher than what's found in db dynamic schema migration willbe attempted."- 1)+ 2)++(defn- add-personal-owner-id-column!+ "Migration 2: Add `personal_owner_id` column to index tables for SQL-level personal collection filtering."+ [tx index-metadata]+ (let [index-metadata (keyword (:metadata-table-name index-metadata))+ execute! (fn [q] (jdbc/execute! tx (sql/format q)))+ table-names (->> (execute! {:select-distinct [:table_name]+ :from [index-metadata]+ :where [[:< :index_version 2]]})+ (mapcat vals))]+ (when (seq table-names)+ (doseq [table-name table-names]+ (execute! {:alter-table [(keyword table-name)]+ :add-column [[:personal_owner_id :int :if-not-exists]]}))+ (execute! {:update index-metadata+ :set {:index_version 2}+ :where [[:in :table_name (vec table-names)]]}))))(defn migrate-dynamic-schema!"Migrate runtime-managed schema, ie. schema of `index_table_...` tables. Migration author is responsible for removingleftovers if necessary."- [_tx {_index-metadata :index-metadata _embedding-model :embedding-model :as _opts}]+ [tx {:keys [index-metadata] :as _opts}];; migration 1: all tables dropped in schema migration in single function call- )+ ;; migration 2: add personal_owner_id column to index tables+ (add-personal-owner-id-column! tx index-metadata))diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/index.cljindex 9004220c1c7..94cf79185fe 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj@@ -2,6 +2,7 @@(:require[buddy.core.codecs :as buddy-codecs][buddy.core.hash :as buddy-hash]+ [clojure.set :as set][clojure.string :as str][com.climate.claypoole :as cp][honey.sql :as sql]@@ -53,6 +54,7 @@[:model :text :not-null][:model_id :text :not-null][:collection_id :int]+ [:personal_owner_id :int][:creator_id :int][:database_id :int][:last_editor_id :int]@@ -115,14 +117,43 @@(= 1 b) true:else (throw (ex-info "Unexpected boolean value" {:v b}))))+(defn- batch-resolve-personal-owner-ids+ "Given a seq of collection-ids, return a map of collection-id -> personal_owner_id.+ Collections not in any personal tree will be absent from the map.+ Uses at most 2 queries regardless of the number of collection-ids."+ [collection-ids]+ (when-let [collection-ids (not-empty (set (remove nil? collection-ids)))]+ (let [colls (t2/select [:model/Collection :id :personal_owner_id :location]+ :id [:in collection-ids])+ {personal true+ non-personal false} (group-by (comp some? :personal_owner_id) colls)+ direct (into {} (map (juxt :id :personal_owner_id)) personal)+ roots (keep (fn [{:keys [id location]}]+ (when-let [root-id (some-> location (str/split #"/" 3) second parse-long)]+ [id root-id]))+ non-personal)+ unknown-roots (set/difference (set (map second roots)) (set (keys direct)))+ root-id->owner (into direct+ (map (juxt :id :personal_owner_id))+ (when (seq unknown-roots)+ (t2/select [:model/Collection :id :personal_owner_id]+ :id [:in unknown-roots]+ :personal_owner_id [:not= nil])))]+ (into direct+ (keep (fn [[coll-id root-id]]+ (when-let [owner (get root-id->owner root-id)]+ [coll-id owner])))+ roots))))+(defn- doc->db-record"Convert a document to a database record with a provided embedding."- [embedding-vec {:keys [model id searchable_text embeddable_text native_query created_at creator_id updated_at- last_editor_id archived verified official_collection database_id collection_id display_type legacy_input- pinned dashboardcard_count view_count last_viewed_at] :as doc}]+ [owner-ids {:keys [model id embedding searchable_text embeddable_text native_query created_at creator_id updated_at+ last_editor_id archived verified official_collection database_id collection_id display_type legacy_input+ pinned dashboardcard_count view_count last_viewed_at] :as doc}]{:model model:model_id id:collection_id collection_id+ :personal_owner_id (get owner-ids collection_id):creator_id creator_id:database_id database_id:last_editor_id last_editor_id@@ -139,8 +170,8 @@:model_updated_at (some-> updated_at to-instant):last_viewed_at (some-> last_viewed_at to-instant):legacy_input [:cast (json/encode legacy_input) :jsonb]- :metadata [:cast (json/encode doc) :jsonb]- :embedding [:raw (format-embedding embedding-vec)]+ :metadata [:cast (json/encode (dissoc doc :embedding)) :jsonb]+ :embedding [:raw (format-embedding embedding)]:text_search_vector (if (:name doc)[:||(search/weighted-tsvector "A" (:name doc))@@ -177,14 +208,14 @@(log/warn e "Failed to set :metabase-search/semantic-index-size metric"))))(defn- batch-update!- [connectable table-name records->sql documents embeddings]+ [connectable table-name records->sql documents](when (seq documents)(u/prog1 (->> documents (map :model) frequencies)(u/profile (str "Semantic index database update of " (count documents) " documents " <>)- (doseq [batch (->> (map vector documents embeddings)- (map (fn [[doc embedding]] (doc->db-record embedding doc)))- (partition-all *batch-size*))]- (jdbc/execute! connectable (records->sql batch)))+ (let [owner-ids (batch-resolve-personal-owner-ids (map :collection_id documents))]+ (doseq [batch (->> (map #(doc->db-record owner-ids %) documents)+ (partition-all *batch-size*))]+ (jdbc/execute! connectable (records->sql batch))))(analytics-set-index-size! connectable table-name)))))(defn- execute-with-counts [connectable model ids sql]@@ -247,7 +278,7 @@(let [table-name (or table-name (model-table-name embedding-model))]{:embedding-model embedding-model:table-name table-name- :version 1}))+ :version 2}))(defn- upsert-embedding!-fn [connectable index text->docs](fn [text->embedding]@@ -269,8 +300,7 @@(sql.helpers/on-conflict :model :model_id)(sql.helpers/do-update-set (db-records->update-set db-records))sql-format-quoted))- batch-documents- (map :embedding batch-documents)))))+ batch-documents))))(defn- unwrap-pgobject[^PGobject obj]@@ -469,11 +499,42 @@(create-index-table-if-not-exists! db index)(jdbc/execute! db ["select table_name from INFORMATION_SCHEMA.tables where table_name like 'index_table_%'"]))+(defn- personal-collection-filter+ "Generate a WHERE condition for personal collection filtering based on the filter type.++ | Filter | Personal | Others' Personal | Shared Coll. | No Coll. |+ |----------------|----------|------------------|--------------|----------|+ | all | ✅ | ✅ | ✅ | ✅ |+ | only-mine | ✅ | ❌ | ❌ | ❌ |+ | only | ✅ | ✅ | ❌ | ❌ |+ | exclude | ❌ | ❌ | ✅ | ✅ |+ | exclude-others | ✅ | ❌ | ✅ | ✅ |"+ [{:keys [filter-items-in-personal-collection current-user-id]}]+ (case (or filter-items-in-personal-collection "all")+ "all"+ nil++ "only-mine"+ [:= :personal_owner_id current-user-id]++ "only"+ [:is-not :personal_owner_id nil]++ "exclude"+ [:is :personal_owner_id nil]++ "exclude-others"+ [:or+ [:is :personal_owner_id nil]+ [:= :personal_owner_id current-user-id]]))+(defn- search-filters"Generate WHERE conditions based on search context filters."- [{:keys [archived? verified models created-at created-by last-edited-at last-edited-by table-db-id ids display-type]}]+ [{:keys [archived? verified models created-at created-by last-edited-at last-edited-by+ table-db-id ids display-type] :as search-context}](let [conditions (filter some?- [(when (some? archived?)+ [(personal-collection-filter search-context)+ (when (some? archived?)[:= :archived archived?])(when (some? verified)[:= :verified verified])@@ -505,6 +566,7 @@[:model :model][:model_id :model_id][:collection_id :collection_id]+ [:personal_owner_id :personal_owner_id][:creator_id :creator_id][:database_id :database_id][:last_editor_id :last_editor_id]@@ -693,90 +755,6 @@result))-(defn- get-personal-collection-ids- "Get the set of personal collection IDs by extracting root collection IDs from locations."- [collections-map]- (let [root-collection-ids (->> collections-map- vals- (map :location)- (keep (fn [location]- (when-let [match (re-find #"^/(\d+)/" location)]- (parse-long (second match)))))- distinct)]- (when (seq root-collection-ids)- (->> (t2/select [:collection :id]- :id [:in root-collection-ids]- :personal_owner_id [:not= nil])- (map :id)- set))))--(defn- filter-by-collection- "Filter documents based on personal collection preferences.- Equivalent to metabase.search.filter/personal-collections-where-clause but operates on docs in memory.-- | Filter | Personal | Others' Personal | Shared Coll. | No Coll. |- |----------------|----------|------------------|--------------|----------|- | all | ✅ | ✅ | ✅ | ✅ |- | only-mine | ✅ | ❌ | ❌ | ❌ |- | only | ✅ | ✅ | ❌ | ❌ |- | exclude | ❌ | ❌ | ✅ | ✅ |- | exclude-others | ✅ | ❌ | ✅ | ✅ |- "- [docs {:keys [filter-items-in-personal-collection current-user-id]}]- (let [filter-type (or filter-items-in-personal-collection "all")]- (if (= filter-type "all")- docs- (let [collection-ids (keep :collection_id docs)- collections-map (when (seq collection-ids)- (->> (t2/select [:collection :id :location :personal_owner_id]- :id [:in collection-ids])- (into {} (map (juxt :id identity)))))- personal-collection-ids (when (not= filter-type "only-mine")- (get-personal-collection-ids collections-map))- user-personal-collection-id (t2/select-one-pk :model/Collection :personal_owner_id [:= current-user-id])- is-personal-collection? (fn [coll] (some? (:personal_owner_id coll)))- is-owned-by-user? (fn [coll] (= (:personal_owner_id coll) current-user-id))- is-in-user-personal-tree? (fn [coll] (str/starts-with? (:location coll) (str "/" user-personal-collection-id "/")))- is-in-any-personal-tree? (fn [coll]- (when-let [match (re-find #"^/(\d+)/" (:location coll))]- (contains? personal-collection-ids (parse-long (second match)))))]- (case filter-type- "only-mine"- (filterv (fn [doc]- (when-let [collection (get collections-map (:collection_id doc))]- (or (is-owned-by-user? collection)- (is-in-user-personal-tree? collection))))- docs)-- "only"- (filterv (fn [doc]- (when-let [collection (get collections-map (:collection_id doc))]- (or (is-personal-collection? collection)- (is-in-any-personal-tree? collection))))- docs)-- "exclude"- (filterv (fn [doc]- (let [collection-id (:collection_id doc)]- (or (nil? collection-id)- (when-let [collection (get collections-map collection-id)]- (and (not (is-personal-collection? collection))- (not (is-in-any-personal-tree? collection)))))))- docs)-- "exclude-others"- (filterv (fn [doc]- (let [collection-id (:collection_id doc)]- (or (nil? collection-id)- (when-let [collection (get collections-map collection-id)]- (or (is-owned-by-user? collection)- (is-in-user-personal-tree? collection)- (and (not (is-personal-collection? collection))- (not (is-in-any-personal-tree? collection))))))))- docs)-- docs)))))-(defn- filter-by-collection-id"Filter documents by collection and all descendant collections."[docs collection-id]@@ -793,23 +771,6 @@(str/starts-with? (:location collection) (str "/" collection-id "/")))))))docs)))-(defn- apply-collection-filter- "Apply personal collection filtering with logging."- [search-context docs]- (let [filter-type (:filter-items-in-personal-collection search-context)]- (if (or (nil? filter-type) (= filter-type "all"))- docs- (let [timer (u/start-timer)- filtered-docs (filter-by-collection docs search-context)- time-ms (u/since-ms timer)]- (log/debug "Collection filter" {:filter filter-type- :before (count docs)- :after (count filtered-docs)- :dropped (- (count docs) (count filtered-docs))- :time_ms time-ms})- (analytics/inc! :metabase-search/semantic-collection-filter-ms time-ms)- filtered-docs))))-(defn- apply-collection-id-filter"Apply collection ID filtering with logging."[search-context docs]@@ -858,7 +819,6 @@filter-timer (u/start-timer)filtered-results (->> raw-resultsfilter-read-permitted- (apply-collection-filter search-context)(apply-collection-id-filter search-context)(mapv search/collapse-id))filter-time-ms (u/since-ms filter-timer)diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.cljindex 3e25c41ffb3..09b1e7546bc 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj@@ -371,7 +371,6 @@(is (contains? metric-names :metabase-search/semantic-embedding-ms))(is (contains? metric-names :metabase-search/semantic-db-query-ms))(is (contains? metric-names :metabase-search/semantic-permission-filter-ms))- (is (contains? metric-names :metabase-search/semantic-collection-filter-ms))(is (contains? metric-names :metabase-search/semantic-appdb-scores-ms)))(testing "timing values are reasonable"@@ -380,10 +379,8 @@:metabase-search/semantic-embedding-ms:metabase-search/semantic-db-query-ms:metabase-search/semantic-permission-filter-ms- :metabase-search/semantic-collection-filter-ms:metabase-search/semantic-appdb-scores-ms} metric)](let [time-ms (if (#{:metabase-search/semantic-permission-filter-ms- :metabase-search/semantic-collection-filter-ms:metabase-search/semantic-appdb-scores-ms}metric)(first args)@@ -400,10 +397,43 @@(is (map? labels) "Should have labels map")(is (contains? labels :embedding-model) "Should include embedding-model label")))))))))))-(deftest filter-by-collection-test- (testing "filter-by-collection function filters documents based on personal collection preferences"- (let [user1-id (mt/user->id :rasta)- user2-id (mt/user->id :crowberto)+(deftest personal-collection-filter-test+ (testing "personal-collection-filter generates correct SQL WHERE clauses"+ (let [user-id 42]+ (testing "filter-type 'all' returns nil (no filter)"+ (is (nil? (#'semantic.index/personal-collection-filter+ {:filter-items-in-personal-collection "all" :current-user-id user-id}))))++ (testing "filter-type nil defaults to 'all' behavior"+ (is (nil? (#'semantic.index/personal-collection-filter+ {:filter-items-in-personal-collection nil :current-user-id user-id}))))++ (testing "filter-type 'only-mine' returns only current user's personal collection items"+ (is (= [:= :personal_owner_id user-id]+ (#'semantic.index/personal-collection-filter+ {:filter-items-in-personal-collection "only-mine" :current-user-id user-id}))))++ (testing "filter-type 'only' returns all personal collection items (any user)"+ (is (= [:is-not :personal_owner_id nil]+ (#'semantic.index/personal-collection-filter+ {:filter-items-in-personal-collection "only" :current-user-id user-id}))))++ (testing "filter-type 'exclude' returns only shared and uncollected items"+ (is (= [:is :personal_owner_id nil]+ (#'semantic.index/personal-collection-filter+ {:filter-items-in-personal-collection "exclude" :current-user-id user-id}))))++ (testing "filter-type 'exclude-others' returns user's personal items plus shared/uncollected items"+ (is (= [:or+ [:is :personal_owner_id nil]+ [:= :personal_owner_id user-id]]+ (#'semantic.index/personal-collection-filter+ {:filter-items-in-personal-collection "exclude-others" :current-user-id user-id})))))))++(deftest batch-resolve-personal-owner-ids-test+ (testing "batch-resolve-personal-owner-ids correctly resolves personal collection ownership"+ (let [user1-id (mt/user->id :rasta)+ user2-id (mt/user->id :crowberto)user1-personal-coll-id (u/the-id (collection/user->personal-collection user1-id))user2-personal-coll-id (u/the-id (collection/user->personal-collection user2-id))](mt/with-temp@@ -411,48 +441,30 @@:model/Collection {user1-sub-coll-id :id} {:location (str "/" user1-personal-coll-id "/") :name "User1 Sub"}:model/Collection {user2-sub-coll-id :id} {:location (str "/" user2-personal-coll-id "/") :name "User2 Sub"}]- (let [docs [{:id "doc1" :model "card" :collection_id user1-personal-coll-id}- {:id "doc2" :model "card" :collection_id user2-personal-coll-id}- {:id "doc3" :model "card" :collection_id shared-coll-id}- {:id "doc4" :model "card" :collection_id nil}- {:id "doc5" :model "card" :collection_id user1-sub-coll-id}- {:id "doc6" :model "card" :collection_id user2-sub-coll-id}]]-- (testing "filter-type 'all' returns all documents unchanged"- (let [context {:filter-items-in-personal-collection "all" :current-user-id user1-id}- result (#'semantic.index/filter-by-collection docs context)]- (is (= 6 (count result)))- (is (= (set (map :id docs)) (set (map :id result))))))-- (testing "filter-type nil defaults to 'all' behavior"- (let [context {:filter-items-in-personal-collection nil :current-user-id user1-id}- result (#'semantic.index/filter-by-collection docs context)]- (is (= 6 (count result)))- (is (= (set (map :id docs)) (set (map :id result))))))-- (testing "filter-type 'only-mine' returns only current user's personal collection items"- (let [context {:filter-items-in-personal-collection "only-mine" :current-user-id user1-id}- result (#'semantic.index/filter-by-collection docs context)- result-ids (set (map :id result))]- (is (= #{"doc1" "doc5"} result-ids))))-- (testing "filter-type 'only' returns all personal collection items (any user)"- (let [context {:filter-items-in-personal-collection "only" :current-user-id user1-id}- result (#'semantic.index/filter-by-collection docs context)- result-ids (set (map :id result))]- (is (= #{"doc1" "doc2" "doc5" "doc6"} result-ids))))-- (testing "filter-type 'exclude' returns only shared and uncollected items"- (let [context {:filter-items-in-personal-collection "exclude" :current-user-id user1-id}- result (#'semantic.index/filter-by-collection docs context)- result-ids (set (map :id result))]- (is (= #{"doc3" "doc4"} result-ids))))-- (testing "filter-type 'exclude-others' returns user's personal items plus shared/uncollected items"- (let [context {:filter-items-in-personal-collection "exclude-others" :current-user-id user1-id}- result (#'semantic.index/filter-by-collection docs context)- result-ids (set (map :id result))]- (is (= #{"doc1" "doc3" "doc4" "doc5"} result-ids)))))))))+ (testing "empty input returns empty map"+ (is (empty? (#'semantic.index/batch-resolve-personal-owner-ids [])))+ (is (empty? (#'semantic.index/batch-resolve-personal-owner-ids [nil]))))++ (testing "shared collection is absent from result"+ (is (empty? (#'semantic.index/batch-resolve-personal-owner-ids [shared-coll-id]))))++ (testing "personal collections map to their owners"+ (is (= {user1-personal-coll-id user1-id+ user2-personal-coll-id user2-id}+ (#'semantic.index/batch-resolve-personal-owner-ids+ [user1-personal-coll-id user2-personal-coll-id]))))++ (testing "sub-collections of personal collections map to root personal owner"+ (is (= {user1-sub-coll-id user1-id+ user2-sub-coll-id user2-id}+ (#'semantic.index/batch-resolve-personal-owner-ids+ [user1-sub-coll-id user2-sub-coll-id]))))++ (testing "mixed input resolves all in one call"+ (is (= {user1-personal-coll-id user1-id+ user2-sub-coll-id user2-id}+ (#'semantic.index/batch-resolve-personal-owner-ids+ [user1-personal-coll-id user2-sub-coll-id shared-coll-id nil]))))))))(deftest filter-can-read-indexed-entity-test(mt/with-premium-features #{:semantic-search}@@ -540,7 +552,8 @@:id "123":searchable_text "test content":embeddable_text "test content"- :creator_id 1}]+ :creator_id 1+ :embedding embedding-vec}](testing "MySQL-style integer booleans are converted to real booleans"(let [doc-with-mysql-booleans (assoc base-doc@@ -548,7 +561,7 @@:official_collection 1:pinned 0:verified 1)- result (#'semantic.index/doc->db-record embedding-vec doc-with-mysql-booleans)]+ result (#'semantic.index/doc->db-record nil doc-with-mysql-booleans)](is (false? (:archived result)))(is (true? (:official_collection result)))(is (false? (:pinned result)))@@ -560,7 +573,7 @@:official_collection false:pinned true:verified false)- result (#'semantic.index/doc->db-record embedding-vec doc-with-real-booleans)]+ result (#'semantic.index/doc->db-record nil doc-with-real-booleans)](is (true? (:archived result)))(is (false? (:official_collection result)))(is (true? (:pinned result)))@@ -568,7 +581,7 @@(testing "nil boolean fields are handled correctly"(let [doc-with-nil-booleans base-doc- result (#'semantic.index/doc->db-record embedding-vec doc-with-nil-booleans)]+ result (#'semantic.index/doc->db-record nil doc-with-nil-booleans)](is (nil? (:archived result)))(is (nil? (:official_collection result)))(is (nil? (:pinned result)))diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/migration_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/migration_test.cljindex abba02dfedc..e7d98456725 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/migration_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/migration_test.clj@@ -28,8 +28,9 @@(semantic.tu/with-test-db-defaults!(letfn [(migrate-and-get-db-version[attempted-version]- (with-redefs [semantic.db.migration.impl/schema-version attempted-version- semantic.db.migration.impl/migrate-schema! (constantly nil)]+ (with-redefs [semantic.db.migration.impl/schema-version attempted-version+ semantic.db.migration.impl/migrate-schema! (constantly nil)+ semantic.db.migration.impl/migrate-dynamic-schema! (constantly nil)](log.capture/with-log-messages-for-level [messages [metabase-enterprise.semantic-search.db.migration :info]](semantic.db.connection/with-migrate-tx [tx](semantic.db.migration/maybe-migrate! tx nil)@@ -190,6 +191,7 @@"model_updated_at""name""official_collection"+ "personal_owner_id""pinned""text_search_vector""text_search_with_native_query_vector"