The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Semantic search performance improvements (#72834)
Tyler Plude5faf92710935
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex f5ceab781c0..920b68cd18e 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -3194,7 +3194,7 @@tasktracingutil}- :model-imports #{:model/Card :model/Collection :model/ModelIndex}}+ :model-imports #{:model/Card :model/Collection :model/Dashboard}}enterprise/serialization{:team "Graphy"diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/index.cljindex aa4011188b4..93b0743a5ec 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj@@ -585,7 +585,7 @@[:model_created_at :model_created_at][:model_updated_at :model_updated_at][:last_viewed_at :last_viewed_at]- [:metadata :metadata]])+ [:legacy_input :legacy_input]])(defn- keyword-search-query [index search-context](let [filters (search-filters search-context)@@ -691,70 +691,75 @@{:keys [ctes query]} (flatten-ctes hybrid-query)all-ctes (conj ctes [:hybrid_results query])full-query {:with all-ctes- :select [:id :model_id :model :content :verified :metadata :semantic_rank :keyword_rank]+ :select [:id :model_id :model :content :verified :legacy_input :semantic_rank :keyword_rank]:from [[:hybrid_results :search_index]]:limit (semantic-settings/semantic-search-results-limit)}](scoring/with-scores search-context scorers full-query)))(defn- legacy-input-with-score- "Fetches the legacy_input field from a result's metadata and attaches a score based on the- embedding distance."+ "Extracts the (already-decoded) `:legacy_input` map from `row` and attaches the total score+ plus per-scorer breakdown."[weights scorers row]- (let [raw (get-in row [:metadata :legacy_input])- ;; legacy_input may be a pre-encoded JSON string (from ->document) stored as a string- ;; value inside the metadata JSONB blob; decode it back to a map if so.- legacy-input (cond-> raw (string? raw) (json/decode true))]- (assoc legacy-input- :score (:total_score row 1.0)- :all-scores (scoring/all-scores weights scorers row))))--(defn- decode-metadata- "Decode `row`s `:metadata`."+ (assoc (:legacy_input row)+ :score (:total_score row 1.0)+ :all-scores (scoring/all-scores weights scorers row)))++(defn- decode-legacy-input+ "Decode `row`s `:legacy_input` JSONB PGobject into a Clojure map."[row]- (update row :metadata decode-pgobject))--(defn- filter-can-read-indexed-entity- "Check permissions for indexed entities by resolving to their parent model / card"- [indexed-entity-docs]- (let [->model-index-id #(-> % :id search/indexed-entity-id->model-index-id)- model-index-ids (into #{} (map ->model-index-id indexed-entity-docs))- model-indexes (when (seq model-index-ids)- (t2/select :model/ModelIndex :id [:in model-index-ids]))- index-id->card (when (seq model-indexes)- (let [card-ids (map :model_id model-indexes)- cards (t2/select :model/Card :id [:in card-ids])- cards-by-id (u/index-by :id cards)]- (into {}- (map (fn [model-index]- [(:id model-index) (get cards-by-id (:model_id model-index))])- model-indexes))))]- (filterv (fn [doc]- (when-let [model-index-id (-> doc ->model-index-id)]- (when-let [parent-card (get index-id->card model-index-id)]- (mi/can-read? parent-card))))- indexed-entity-docs)))+ (update row :legacy_input decode-pgobject))++;; Search-models whose `mi/can-read?` is a pure function of `:collection_id`, which is+;; already denormalized on the index row. Values are the Toucan model whose `can-read?`+;; defmethod is invoked on a stub instance carrying only `:collection_id`.+;;+;; - "card" / "metric" / "dataset" → :model/Card+;; - "dashboard" → :model/Dashboard (as of 20260420 this _could_ be+;; :model/Card for additional albeit modest perf gains+;; but choosing to not silently ride Card's semantics)+;; - "indexed-entity" → :model/Card by design: the index row's+;; `collection_id` is the parent Card's (denormalized+;; via the spec's Collection join), so routing through+;; Card short-circuits the original ModelIndex → Card+;; resolution in `filter-can-read-indexed-entity`.+(def ^:private collection-id-only-search-models+ {"card" :model/Card+ "metric" :model/Card+ "dataset" :model/Card+ "dashboard" :model/Dashboard+ "indexed-entity" :model/Card})(defn- filter-read-permitted"Returns only those documents in `docs` whose corresponding t2 instances pass an mi/can-read? check for the bound api user."[docs](let [timer (u/start-timer)doc->t2-model (fn [doc] (:model (search/spec (:model doc))))- {indexed-entities true regular-docs false} (group-by #(= "indexed-entity" (:model %)) docs)- other-docs (for [[t2-model docs] (group-by doc->t2-model regular-docs)- t2-instance (t2/select t2-model :id [:in (map :id docs)])]- t2-instance)- permitted-entities (filter-can-read-indexed-entity indexed-entities)- doc->t2 (comp (u/index-by (juxt :id t2/model) other-docs)+ {fast-docs true slow-docs false} (group-by #(contains? collection-id-only-search-models (:model %)) docs)+ ;; Fast path: the permission check depends only on `:collection_id`, which is already on the+ ;; index row. Dedupe by `[stub-model, collection_id]` so `can-read?` runs at most once per+ ;; (model, collection) pair instead of once per document.+ readable? (memoize+ (fn [stub-model coll-id]+ (mi/can-read? (t2/instance stub-model {:collection_id coll-id}))))+ fast-filtered (filterv (fn [doc]+ (readable? (get collection-id-only-search-models (:model doc))+ (:collection_id doc)))+ fast-docs)+ slow-t2-instances (vec+ (for [[t2-model docs] (group-by doc->t2-model slow-docs)+ t2-instance (t2/select t2-model :id [:in (map :id docs)])]+ t2-instance))+ doc->t2 (comp (u/index-by (juxt :id t2/model) slow-t2-instances)(juxt :id doc->t2-model))- result (into- (filterv (fn [doc] (some-> doc doc->t2 mi/can-read?)) regular-docs)- permitted-entities)+ slow-filtered (filterv (fn [doc] (some-> doc doc->t2 mi/can-read?)) slow-docs)+ result (into fast-filtered slow-filtered)time-ms (u/since-ms timer)](log/debug "Permission filtering" {:before-count (count docs):after-count (count result)- :indexed-entities-count (count indexed-entities)- :regular-docs-count (count regular-docs)+ :fast-count (count fast-docs)+ :slow-count (count slow-docs)+ :slow-fetched-count (count slow-t2-instances):time-ms time-ms})(analytics/inc! :metabase-search/semantic-permission-filter-ms time-ms)@@ -819,7 +824,7 @@weights (search.config/weights search-context)scorers (scoring/semantic-scorers (:table-name index) search-context)query (scored-search-query index embedding search-context scorers)- xform (comp (map decode-metadata)+ xform (comp (map decode-legacy-input)(map (partial legacy-input-with-score weights (keys scorers))))reducible (reducible-search-query db query)raw-results (tracing/with-span :search "search.semantic.db-query"diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.cljindex 09b1e7546bc..46feaf74b34 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj@@ -466,74 +466,87 @@(#'semantic.index/batch-resolve-personal-owner-ids[user1-personal-coll-id user2-sub-coll-id shared-coll-id nil]))))))))-(deftest filter-can-read-indexed-entity-test+(deftest filter-read-permitted-indexed-entity-test(mt/with-premium-features #{:semantic-search}- (testing "filter-can-read-indexed-entity function"- (mt/with-temp [:model/Collection {coll-id :id} {}- :model/Card model-1 (assoc (mt/card-with-source-metadata-for-query- (mt/mbql-query products {:fields [$id $title]- :limit 1}))- :type "model"- :name "Readable Model"- :database_id (mt/id)- :collection_id coll-id)- :model/Card model-2 (assoc (mt/card-with-source-metadata-for-query- (mt/mbql-query products {:fields [$id $title]- :limit 1}))- :type "model"- :name "Unreadable Model"- :database_id (mt/id)- :collection_id coll-id)- :model/ModelIndex model-index-1 {:model_id (:id model-1)- :pk_ref (mt/$ids :products $id)- :value_ref (mt/$ids :products $title)- :schedule "0 0 0 * * *"- :state "initial"- :creator_id (mt/user->id :rasta)}- :model/ModelIndex model-index-2 {:model_id (:id model-2)- :pk_ref (mt/$ids :products $id)- :value_ref (mt/$ids :products $title)- :schedule "0 0 0 * * *"- :state "initial"- :creator_id (mt/user->id :rasta)}]- (let [indexed-entity-docs [{:id (str (:id model-index-1) ":123")+ (testing "filter-read-permitted routes indexed-entity docs through the collection_id-only fast path"+ (mt/with-temp [:model/Collection {readable-coll-id :id} {}+ :model/Collection {unreadable-coll-id :id} {}]+ (let [indexed-entity-docs [{:id "1:123":model "indexed-entity"- :content "Test Entity 1"}- {:id (str (:id model-index-2) ":456")+ :collection_id readable-coll-id}+ {:id "2:456":model "indexed-entity"- :content "Test Entity 2"}- {:id "invalid:789"+ :collection_id unreadable-coll-id}+ {:id "3:789":model "indexed-entity"- :content "Invalid Entity"}]]+ :collection_id nil}]]- (testing "returns all entities when user can read all parent cards"+ (testing "keeps all entities when user has root permissions"(binding [api/*current-user-permissions-set* (atom #{"/"})]- (let [result (#'semantic.index/filter-can-read-indexed-entity indexed-entity-docs)]- (is (= 2 (count result)))- (is (= #{(str (:id model-index-1) ":123") (str (:id model-index-2) ":456")}- (set (map :id result)))))))+ (let [result (#'semantic.index/filter-read-permitted indexed-entity-docs)]+ (is (= 3 (count result))))))- (testing "returns no entities when user cannot read any parent cards"+ (testing "drops all entities when user has no permissions"(binding [api/*current-user-permissions-set* (atom #{})]- (let [result (#'semantic.index/filter-can-read-indexed-entity indexed-entity-docs)]+ (let [result (#'semantic.index/filter-read-permitted indexed-entity-docs)](is (= 0 (count result))))))- (testing "returns subset when user can read some parent cards"- (with-redefs [mi/can-read? (fn [card] (= (:id card) (:id model-1)))]- (let [result (#'semantic.index/filter-can-read-indexed-entity indexed-entity-docs)]+ (testing "keeps only entities in readable collections"+ (binding [api/*current-user-permissions-set* (atom #{(format "/collection/%d/read/" readable-coll-id)})]+ (let [result (#'semantic.index/filter-read-permitted indexed-entity-docs)](is (= 1 (count result)))- (is (= (str (:id model-index-1) ":123") (:id (first result)))))))+ (is (= "1:123" (:id (first result)))))))++ (testing "memoizes permission check per collection_id across docs"+ (let [calls (atom 0)+ real-can-read? mi/can-read?]+ (with-redefs [mi/can-read? (fn [& args]+ (swap! calls inc)+ (apply real-can-read? args))]+ (binding [api/*current-user-permissions-set* (atom #{"/"})]+ (#'semantic.index/filter-read-permitted+ (repeat 50 {:id "1:1" :model "indexed-entity" :collection_id readable-coll-id}))+ (is (= 1 @calls) "expected one can-read? call for the single distinct collection_id")))))(testing "handles empty input"- (is (= [] (#'semantic.index/filter-can-read-indexed-entity []))))+ (is (= [] (#'semantic.index/filter-read-permitted [])))))))))- (testing "handles invalid entity IDs"- (let [invalid-only-docs [{:id "invalid:789" :model "indexed-entity" :content "Invalid"}]]- (is (= [] (#'semantic.index/filter-can-read-indexed-entity invalid-only-docs)))))-- (testing "handles non-existent model indexes"- (let [non-existent-docs [{:id "99999:123" :model "indexed-entity" :content "Non-existent"}]]- (is (= [] (#'semantic.index/filter-can-read-indexed-entity non-existent-docs))))))))))+(deftest filter-read-permitted-dispatch-test+ (mt/with-premium-features #{:semantic-search}+ (testing "fast path dispatches through the per-search-model t2 model"+ (mt/with-temp [:model/Collection {coll-id :id} {}]+ (binding [api/*current-user-permissions-set* (atom #{"/"})]++ (testing "card and dashboard in the same collection do NOT share a memo bucket"+ ;; Card dispatches through :model/Card, Dashboard through :model/Dashboard. Their+ ;; `can-read?` answers happen to agree today (both route through+ ;; `can-read-audit-helper` which ignores the model keyword for non-Collection models),+ ;; but the dispatch MUST still hit each model's own defmethod so future divergence+ ;; cannot silently ride on Card's semantics.+ (let [calls (atom 0)+ real-can-read? mi/can-read?]+ (with-redefs [mi/can-read? (fn [& args]+ (swap! calls inc)+ (apply real-can-read? args))]+ (#'semantic.index/filter-read-permitted+ [{:id 1 :model "card" :collection_id coll-id}+ {:id 2 :model "dashboard" :collection_id coll-id}])+ (is (= 2 @calls)+ "card and dashboard must dispatch separately even for the same collection_id"))))++ (testing "card and indexed-entity share the :model/Card memo bucket"+ ;; indexed-entity is deliberately routed through :model/Card — its index row's+ ;; `collection_id` is the parent Card's, denormalized at ingest time.+ (let [calls (atom 0)+ real-can-read? mi/can-read?]+ (with-redefs [mi/can-read? (fn [& args]+ (swap! calls inc)+ (apply real-can-read? args))]+ (#'semantic.index/filter-read-permitted+ [{:id 1 :model "card" :collection_id coll-id}+ {:id "1:99" :model "indexed-entity" :collection_id coll-id}])+ (is (= 1 @calls)+ "card and indexed-entity both dispatch through :model/Card and memoize together")))))))))(deftest to-boolean-test(testing "to-boolean function correctly converts various input types to booleans"diff --git a/src/metabase/search/impl.clj b/src/metabase/search/impl.cljindex 1d43fefa3c9..6e44871d8c1 100644--- a/src/metabase/search/impl.clj+++ b/src/metabase/search/impl.clj@@ -386,37 +386,14 @@(cond-> (t2/instance-of? :model/Collection instance) map-collection))))(defn- add-can-write [search-ctx row]- (if (some #(mi/instance-of? % row) [:model/Dashboard :model/Card])+ (if (some #(mi/instance-of? % row) [:model/Dashboard :model/Card :model/Collection])(assoc row :can_write (can-write? search-ctx row))row))(defn- normalize-result-more- "Additional normalization that is done after we've filtered by permissions, as its more expensive."- [search-ctx result]- (->> (update result :pk_ref json/decode)- (add-can-write search-ctx)))--(defn- search-results [search-ctx model-set-fn total-results]- (let [add-perms-for-col (fn [item]- (cond-> item- (mi/instance-of? :model/Collection item)- (assoc :can_write (can-write? search-ctx item))))]- ;; We get to do this slicing and dicing with the result data because- ;; the pagination of search is for UI improvement, not for performance.- ;; We intend for the cardinality of the search results to be below the default max before this slicing occurs- (cond-> {:data (cond->> total-results- (some? (:offset-int search-ctx)) (drop (:offset-int search-ctx))- (some? (:limit-int search-ctx)) (take (:limit-int search-ctx))- true (map add-perms-for-col))- :limit (:limit-int search-ctx)- :models (:models search-ctx)- :offset (:offset-int search-ctx)- :table_db_id (:table-db-id search-ctx)- :engine (:search-engine search-ctx)- :total (count total-results)}-- (:calculate-available-models? search-ctx)- (assoc :available_models (model-set-fn search-ctx)))))+ "Additional normalization that is done after we've filtered by permissions."+ [_search-ctx result]+ (update result :pk_ref json/decode))(defn- hydrate-dashboards [results](->> (t2/hydrate results [:dashboard :moderation_status])@@ -438,6 +415,34 @@item))search-results)))+(defn- search-results [search-ctx model-set-fn ranked-results]+ ;; Slice the ranked window down to the requested page *before* hydration/serialization so+ ;; display-only work (including the expensive per-row `:can_write` permission check) only runs+ ;; on the rows we actually return. The pagination of search is for UI improvement, not for+ ;; performance — the ranked set is already capped by `max-filtered-results` — but doing the+ ;; slice here lets downstream steps amortize their batch hydrations over ~limit rows instead+ ;; of the full cap.+ (let [paginated-results (cond->> ranked-results+ (some? (:offset-int search-ctx)) (drop (:offset-int search-ctx))+ (some? (:limit-int search-ctx)) (take (:limit-int search-ctx))+ true hydrate-dashboards+ true hydrate-user-metadata+ (:include-metadata? search-ctx) (add-metadata)+ (:model-ancestors? search-ctx) (add-dataset-collection-hierarchy)+ true (add-collection-effective-location)+ true (map (partial add-can-write search-ctx))+ true (map serialize))]+ (cond-> {:data paginated-results+ :limit (:limit-int search-ctx)+ :models (:models search-ctx)+ :offset (:offset-int search-ctx)+ :table_db_id (:table-db-id search-ctx)+ :engine (:search-engine search-ctx)+ :total (count ranked-results)}++ (:calculate-available-models? search-ctx)+ (assoc :available_models (model-set-fn search-ctx)))))+(mu/defn search"Builds a search query that includes all the searchable entities, and runs it."[search-ctx :- search.config/SearchContext]@@ -452,11 +457,5 @@(filter (partial check-permissions-for-model search-ctx))(map (partial normalize-result-more search-ctx))(keep #(search.engine/score scoring-ctx %)))- total-results (cond->> (scoring/top-results reducible-results search.config/max-filtered-results xf)- true hydrate-dashboards- true hydrate-user-metadata- (:include-metadata? search-ctx) (add-metadata)- (:model-ancestors? search-ctx) (add-dataset-collection-hierarchy)- true (add-collection-effective-location)- true (map serialize))]- (search-results search-ctx search.engine/model-set total-results))))+ ranked-results (scoring/top-results reducible-results search.config/max-filtered-results xf)]+ (search-results search-ctx search.engine/model-set ranked-results))))