Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Semantic search performance improvements (#72834)

Tyler Plude5faf92710935
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index f5ceab781c0..920b68cd18e 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -3194,7 +3194,7 @@
task
tracing
util}
- :model-imports #{:model/Card :model/Collection :model/ModelIndex}}
+ :model-imports #{:model/Card :model/Collection :model/Dashboard}}
enterprise/serialization
{:team "Graphy"
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj
index aa4011188b4..93b0743a5ec 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj
@@ -585,7 +585,7 @@
[:model_created_at :model_created_at]
[:model_updated_at :model_updated_at]
[:last_viewed_at :last_viewed_at]
- [:metadata :metadata]])
+ [:legacy_input :legacy_input]])
(defn- keyword-search-query [index search-context]
(let [filters (search-filters search-context)
@@ -691,70 +691,75 @@
{:keys [ctes query]} (flatten-ctes hybrid-query)
all-ctes (conj ctes [:hybrid_results query])
full-query {:with all-ctes
- :select [:id :model_id :model :content :verified :metadata :semantic_rank :keyword_rank]
+ :select [:id :model_id :model :content :verified :legacy_input :semantic_rank :keyword_rank]
:from [[:hybrid_results :search_index]]
:limit (semantic-settings/semantic-search-results-limit)}]
(scoring/with-scores search-context scorers full-query)))
(defn- legacy-input-with-score
- "Fetches the legacy_input field from a result's metadata and attaches a score based on the
- embedding distance."
+ "Extracts the (already-decoded) `:legacy_input` map from `row` and attaches the total score
+ plus per-scorer breakdown."
[weights scorers row]
- (let [raw (get-in row [:metadata :legacy_input])
- ;; legacy_input may be a pre-encoded JSON string (from ->document) stored as a string
- ;; value inside the metadata JSONB blob; decode it back to a map if so.
- legacy-input (cond-> raw (string? raw) (json/decode true))]
- (assoc legacy-input
- :score (:total_score row 1.0)
- :all-scores (scoring/all-scores weights scorers row))))
-
-(defn- decode-metadata
- "Decode `row`s `:metadata`."
+ (assoc (:legacy_input row)
+ :score (:total_score row 1.0)
+ :all-scores (scoring/all-scores weights scorers row)))
+
+(defn- decode-legacy-input
+ "Decode `row`s `:legacy_input` JSONB PGobject into a Clojure map."
[row]
- (update row :metadata decode-pgobject))
-
-(defn- filter-can-read-indexed-entity
- "Check permissions for indexed entities by resolving to their parent model / card"
- [indexed-entity-docs]
- (let [->model-index-id #(-> % :id search/indexed-entity-id->model-index-id)
- model-index-ids (into #{} (map ->model-index-id indexed-entity-docs))
- model-indexes (when (seq model-index-ids)
- (t2/select :model/ModelIndex :id [:in model-index-ids]))
- index-id->card (when (seq model-indexes)
- (let [card-ids (map :model_id model-indexes)
- cards (t2/select :model/Card :id [:in card-ids])
- cards-by-id (u/index-by :id cards)]
- (into {}
- (map (fn [model-index]
- [(:id model-index) (get cards-by-id (:model_id model-index))])
- model-indexes))))]
- (filterv (fn [doc]
- (when-let [model-index-id (-> doc ->model-index-id)]
- (when-let [parent-card (get index-id->card model-index-id)]
- (mi/can-read? parent-card))))
- indexed-entity-docs)))
+ (update row :legacy_input decode-pgobject))
+
+;; Search-models whose `mi/can-read?` is a pure function of `:collection_id`, which is
+;; already denormalized on the index row. Values are the Toucan model whose `can-read?`
+;; defmethod is invoked on a stub instance carrying only `:collection_id`.
+;;
+;; - "card" / "metric" / "dataset" → :model/Card
+;; - "dashboard" → :model/Dashboard (as of 20260420 this _could_ be
+;; :model/Card for additional albeit modest perf gains
+;; but choosing to not silently ride Card's semantics)
+;; - "indexed-entity" → :model/Card by design: the index row's
+;; `collection_id` is the parent Card's (denormalized
+;; via the spec's Collection join), so routing through
+;; Card short-circuits the original ModelIndex → Card
+;; resolution in `filter-can-read-indexed-entity`.
+(def ^:private collection-id-only-search-models
+ {"card" :model/Card
+ "metric" :model/Card
+ "dataset" :model/Card
+ "dashboard" :model/Dashboard
+ "indexed-entity" :model/Card})
(defn- filter-read-permitted
"Returns only those documents in `docs` whose corresponding t2 instances pass an mi/can-read? check for the bound api user."
[docs]
(let [timer (u/start-timer)
doc->t2-model (fn [doc] (:model (search/spec (:model doc))))
- {indexed-entities true regular-docs false} (group-by #(= "indexed-entity" (:model %)) docs)
- other-docs (for [[t2-model docs] (group-by doc->t2-model regular-docs)
- t2-instance (t2/select t2-model :id [:in (map :id docs)])]
- t2-instance)
- permitted-entities (filter-can-read-indexed-entity indexed-entities)
- doc->t2 (comp (u/index-by (juxt :id t2/model) other-docs)
+ {fast-docs true slow-docs false} (group-by #(contains? collection-id-only-search-models (:model %)) docs)
+ ;; Fast path: the permission check depends only on `:collection_id`, which is already on the
+ ;; index row. Dedupe by `[stub-model, collection_id]` so `can-read?` runs at most once per
+ ;; (model, collection) pair instead of once per document.
+ readable? (memoize
+ (fn [stub-model coll-id]
+ (mi/can-read? (t2/instance stub-model {:collection_id coll-id}))))
+ fast-filtered (filterv (fn [doc]
+ (readable? (get collection-id-only-search-models (:model doc))
+ (:collection_id doc)))
+ fast-docs)
+ slow-t2-instances (vec
+ (for [[t2-model docs] (group-by doc->t2-model slow-docs)
+ t2-instance (t2/select t2-model :id [:in (map :id docs)])]
+ t2-instance))
+ doc->t2 (comp (u/index-by (juxt :id t2/model) slow-t2-instances)
(juxt :id doc->t2-model))
- result (into
- (filterv (fn [doc] (some-> doc doc->t2 mi/can-read?)) regular-docs)
- permitted-entities)
+ slow-filtered (filterv (fn [doc] (some-> doc doc->t2 mi/can-read?)) slow-docs)
+ result (into fast-filtered slow-filtered)
time-ms (u/since-ms timer)]
(log/debug "Permission filtering" {:before-count (count docs)
:after-count (count result)
- :indexed-entities-count (count indexed-entities)
- :regular-docs-count (count regular-docs)
+ :fast-count (count fast-docs)
+ :slow-count (count slow-docs)
+ :slow-fetched-count (count slow-t2-instances)
:time-ms time-ms})
(analytics/inc! :metabase-search/semantic-permission-filter-ms time-ms)
@@ -819,7 +824,7 @@
weights (search.config/weights search-context)
scorers (scoring/semantic-scorers (:table-name index) search-context)
query (scored-search-query index embedding search-context scorers)
- xform (comp (map decode-metadata)
+ xform (comp (map decode-legacy-input)
(map (partial legacy-input-with-score weights (keys scorers))))
reducible (reducible-search-query db query)
raw-results (tracing/with-span :search "search.semantic.db-query"
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj
index 09b1e7546bc..46feaf74b34 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/index_test.clj
@@ -466,74 +466,87 @@
(#'semantic.index/batch-resolve-personal-owner-ids
[user1-personal-coll-id user2-sub-coll-id shared-coll-id nil]))))))))
-(deftest filter-can-read-indexed-entity-test
+(deftest filter-read-permitted-indexed-entity-test
(mt/with-premium-features #{:semantic-search}
- (testing "filter-can-read-indexed-entity function"
- (mt/with-temp [:model/Collection {coll-id :id} {}
- :model/Card model-1 (assoc (mt/card-with-source-metadata-for-query
- (mt/mbql-query products {:fields [$id $title]
- :limit 1}))
- :type "model"
- :name "Readable Model"
- :database_id (mt/id)
- :collection_id coll-id)
- :model/Card model-2 (assoc (mt/card-with-source-metadata-for-query
- (mt/mbql-query products {:fields [$id $title]
- :limit 1}))
- :type "model"
- :name "Unreadable Model"
- :database_id (mt/id)
- :collection_id coll-id)
- :model/ModelIndex model-index-1 {:model_id (:id model-1)
- :pk_ref (mt/$ids :products $id)
- :value_ref (mt/$ids :products $title)
- :schedule "0 0 0 * * *"
- :state "initial"
- :creator_id (mt/user->id :rasta)}
- :model/ModelIndex model-index-2 {:model_id (:id model-2)
- :pk_ref (mt/$ids :products $id)
- :value_ref (mt/$ids :products $title)
- :schedule "0 0 0 * * *"
- :state "initial"
- :creator_id (mt/user->id :rasta)}]
- (let [indexed-entity-docs [{:id (str (:id model-index-1) ":123")
+ (testing "filter-read-permitted routes indexed-entity docs through the collection_id-only fast path"
+ (mt/with-temp [:model/Collection {readable-coll-id :id} {}
+ :model/Collection {unreadable-coll-id :id} {}]
+ (let [indexed-entity-docs [{:id "1:123"
:model "indexed-entity"
- :content "Test Entity 1"}
- {:id (str (:id model-index-2) ":456")
+ :collection_id readable-coll-id}
+ {:id "2:456"
:model "indexed-entity"
- :content "Test Entity 2"}
- {:id "invalid:789"
+ :collection_id unreadable-coll-id}
+ {:id "3:789"
:model "indexed-entity"
- :content "Invalid Entity"}]]
+ :collection_id nil}]]
- (testing "returns all entities when user can read all parent cards"
+ (testing "keeps all entities when user has root permissions"
(binding [api/*current-user-permissions-set* (atom #{"/"})]
- (let [result (#'semantic.index/filter-can-read-indexed-entity indexed-entity-docs)]
- (is (= 2 (count result)))
- (is (= #{(str (:id model-index-1) ":123") (str (:id model-index-2) ":456")}
- (set (map :id result)))))))
+ (let [result (#'semantic.index/filter-read-permitted indexed-entity-docs)]
+ (is (= 3 (count result))))))
- (testing "returns no entities when user cannot read any parent cards"
+ (testing "drops all entities when user has no permissions"
(binding [api/*current-user-permissions-set* (atom #{})]
- (let [result (#'semantic.index/filter-can-read-indexed-entity indexed-entity-docs)]
+ (let [result (#'semantic.index/filter-read-permitted indexed-entity-docs)]
(is (= 0 (count result))))))
- (testing "returns subset when user can read some parent cards"
- (with-redefs [mi/can-read? (fn [card] (= (:id card) (:id model-1)))]
- (let [result (#'semantic.index/filter-can-read-indexed-entity indexed-entity-docs)]
+ (testing "keeps only entities in readable collections"
+ (binding [api/*current-user-permissions-set* (atom #{(format "/collection/%d/read/" readable-coll-id)})]
+ (let [result (#'semantic.index/filter-read-permitted indexed-entity-docs)]
(is (= 1 (count result)))
- (is (= (str (:id model-index-1) ":123") (:id (first result)))))))
+ (is (= "1:123" (:id (first result)))))))
+
+ (testing "memoizes permission check per collection_id across docs"
+ (let [calls (atom 0)
+ real-can-read? mi/can-read?]
+ (with-redefs [mi/can-read? (fn [& args]
+ (swap! calls inc)
+ (apply real-can-read? args))]
+ (binding [api/*current-user-permissions-set* (atom #{"/"})]
+ (#'semantic.index/filter-read-permitted
+ (repeat 50 {:id "1:1" :model "indexed-entity" :collection_id readable-coll-id}))
+ (is (= 1 @calls) "expected one can-read? call for the single distinct collection_id")))))
(testing "handles empty input"
- (is (= [] (#'semantic.index/filter-can-read-indexed-entity []))))
+ (is (= [] (#'semantic.index/filter-read-permitted [])))))))))
- (testing "handles invalid entity IDs"
- (let [invalid-only-docs [{:id "invalid:789" :model "indexed-entity" :content "Invalid"}]]
- (is (= [] (#'semantic.index/filter-can-read-indexed-entity invalid-only-docs)))))
-
- (testing "handles non-existent model indexes"
- (let [non-existent-docs [{:id "99999:123" :model "indexed-entity" :content "Non-existent"}]]
- (is (= [] (#'semantic.index/filter-can-read-indexed-entity non-existent-docs))))))))))
+(deftest filter-read-permitted-dispatch-test
+ (mt/with-premium-features #{:semantic-search}
+ (testing "fast path dispatches through the per-search-model t2 model"
+ (mt/with-temp [:model/Collection {coll-id :id} {}]
+ (binding [api/*current-user-permissions-set* (atom #{"/"})]
+
+ (testing "card and dashboard in the same collection do NOT share a memo bucket"
+ ;; Card dispatches through :model/Card, Dashboard through :model/Dashboard. Their
+ ;; `can-read?` answers happen to agree today (both route through
+ ;; `can-read-audit-helper` which ignores the model keyword for non-Collection models),
+ ;; but the dispatch MUST still hit each model's own defmethod so future divergence
+ ;; cannot silently ride on Card's semantics.
+ (let [calls (atom 0)
+ real-can-read? mi/can-read?]
+ (with-redefs [mi/can-read? (fn [& args]
+ (swap! calls inc)
+ (apply real-can-read? args))]
+ (#'semantic.index/filter-read-permitted
+ [{:id 1 :model "card" :collection_id coll-id}
+ {:id 2 :model "dashboard" :collection_id coll-id}])
+ (is (= 2 @calls)
+ "card and dashboard must dispatch separately even for the same collection_id"))))
+
+ (testing "card and indexed-entity share the :model/Card memo bucket"
+ ;; indexed-entity is deliberately routed through :model/Card — its index row's
+ ;; `collection_id` is the parent Card's, denormalized at ingest time.
+ (let [calls (atom 0)
+ real-can-read? mi/can-read?]
+ (with-redefs [mi/can-read? (fn [& args]
+ (swap! calls inc)
+ (apply real-can-read? args))]
+ (#'semantic.index/filter-read-permitted
+ [{:id 1 :model "card" :collection_id coll-id}
+ {:id "1:99" :model "indexed-entity" :collection_id coll-id}])
+ (is (= 1 @calls)
+ "card and indexed-entity both dispatch through :model/Card and memoize together")))))))))
(deftest to-boolean-test
(testing "to-boolean function correctly converts various input types to booleans"
diff --git a/src/metabase/search/impl.clj b/src/metabase/search/impl.clj
index 1d43fefa3c9..6e44871d8c1 100644
--- a/src/metabase/search/impl.clj
+++ b/src/metabase/search/impl.clj
@@ -386,37 +386,14 @@
(cond-> (t2/instance-of? :model/Collection instance) map-collection))))
(defn- add-can-write [search-ctx row]
- (if (some #(mi/instance-of? % row) [:model/Dashboard :model/Card])
+ (if (some #(mi/instance-of? % row) [:model/Dashboard :model/Card :model/Collection])
(assoc row :can_write (can-write? search-ctx row))
row))
(defn- normalize-result-more
- "Additional normalization that is done after we've filtered by permissions, as its more expensive."
- [search-ctx result]
- (->> (update result :pk_ref json/decode)
- (add-can-write search-ctx)))
-
-(defn- search-results [search-ctx model-set-fn total-results]
- (let [add-perms-for-col (fn [item]
- (cond-> item
- (mi/instance-of? :model/Collection item)
- (assoc :can_write (can-write? search-ctx item))))]
- ;; We get to do this slicing and dicing with the result data because
- ;; the pagination of search is for UI improvement, not for performance.
- ;; We intend for the cardinality of the search results to be below the default max before this slicing occurs
- (cond-> {:data (cond->> total-results
- (some? (:offset-int search-ctx)) (drop (:offset-int search-ctx))
- (some? (:limit-int search-ctx)) (take (:limit-int search-ctx))
- true (map add-perms-for-col))
- :limit (:limit-int search-ctx)
- :models (:models search-ctx)
- :offset (:offset-int search-ctx)
- :table_db_id (:table-db-id search-ctx)
- :engine (:search-engine search-ctx)
- :total (count total-results)}
-
- (:calculate-available-models? search-ctx)
- (assoc :available_models (model-set-fn search-ctx)))))
+ "Additional normalization that is done after we've filtered by permissions."
+ [_search-ctx result]
+ (update result :pk_ref json/decode))
(defn- hydrate-dashboards [results]
(->> (t2/hydrate results [:dashboard :moderation_status])
@@ -438,6 +415,34 @@
item))
search-results)))
+(defn- search-results [search-ctx model-set-fn ranked-results]
+ ;; Slice the ranked window down to the requested page *before* hydration/serialization so
+ ;; display-only work (including the expensive per-row `:can_write` permission check) only runs
+ ;; on the rows we actually return. The pagination of search is for UI improvement, not for
+ ;; performance — the ranked set is already capped by `max-filtered-results` — but doing the
+ ;; slice here lets downstream steps amortize their batch hydrations over ~limit rows instead
+ ;; of the full cap.
+ (let [paginated-results (cond->> ranked-results
+ (some? (:offset-int search-ctx)) (drop (:offset-int search-ctx))
+ (some? (:limit-int search-ctx)) (take (:limit-int search-ctx))
+ true hydrate-dashboards
+ true hydrate-user-metadata
+ (:include-metadata? search-ctx) (add-metadata)
+ (:model-ancestors? search-ctx) (add-dataset-collection-hierarchy)
+ true (add-collection-effective-location)
+ true (map (partial add-can-write search-ctx))
+ true (map serialize))]
+ (cond-> {:data paginated-results
+ :limit (:limit-int search-ctx)
+ :models (:models search-ctx)
+ :offset (:offset-int search-ctx)
+ :table_db_id (:table-db-id search-ctx)
+ :engine (:search-engine search-ctx)
+ :total (count ranked-results)}
+
+ (:calculate-available-models? search-ctx)
+ (assoc :available_models (model-set-fn search-ctx)))))
+
(mu/defn search
"Builds a search query that includes all the searchable entities, and runs it."
[search-ctx :- search.config/SearchContext]
@@ -452,11 +457,5 @@
(filter (partial check-permissions-for-model search-ctx))
(map (partial normalize-result-more search-ctx))
(keep #(search.engine/score scoring-ctx %)))
- total-results (cond->> (scoring/top-results reducible-results search.config/max-filtered-results xf)
- true hydrate-dashboards
- true hydrate-user-metadata
- (:include-metadata? search-ctx) (add-metadata)
- (:model-ancestors? search-ctx) (add-dataset-collection-hierarchy)
- true (add-collection-effective-location)
- true (map serialize))]
- (search-results search-ctx search.engine/model-set total-results))))
+ ranked-results (scoring/top-results reducible-results search.config/max-filtered-results xf)]
+ (search-results search-ctx search.engine/model-set ranked-results))))