Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Search Indexing: Cache card schema upgrades (#68912)

Nathan Voxland6e4d96a68f57
diff --git a/src/metabase/search/core.clj b/src/metabase/search/core.clj
index 1df3c034744..2623ebd8a55 100644
--- a/src/metabase/search/core.clj
+++ b/src/metabase/search/core.clj
@@ -3,6 +3,7 @@
(:require
[metabase.analytics.core :as analytics]
[metabase.analytics.prometheus :as prometheus]
+ [metabase.lib-be.core :as lib-be]
[metabase.search.config :as search.config]
[metabase.search.engine :as search.engine]
[metabase.search.impl :as search.impl]
@@ -86,46 +87,48 @@
[& {:as opts}]
(when (supports-index?)
(log/info "Initializing search indexes")
- ;; If there are multiple indexes, return the peak inserted for each type. In practice, they should all be the same.
- (try
- (let [timer (u/start-timer)
- report (reduce (partial merge-with max)
- nil
- (for [e (search.engine/active-engines)]
- (search.engine/init! e opts)))
- duration (u/since-ms timer)]
- (if (seq report)
- (do
- (analytics/inc! :metabase-search/index-reindex-ms duration)
- (prometheus/observe! :metabase-search/index-reindex-duration-ms duration)
- (doseq [[model cnt] report]
- (analytics/inc! :metabase-search/index-reindexes {:model model} cnt))
- (log/infof "Index initialized in %.0fms %s" duration (sort-by (comp - val) report))
- report)
- (log/info "Found existing search index, and using it.")))
- (catch Exception e
- (analytics/inc! :metabase-search/index-error)
- (throw e)))))
+ (lib-be/with-metadata-provider-cache
+ ;; If there are multiple indexes, return the peak inserted for each type. In practice, they should all be the same.
+ (try
+ (let [timer (u/start-timer)
+ report (reduce (partial merge-with max)
+ nil
+ (for [e (search.engine/active-engines)]
+ (search.engine/init! e opts)))
+ duration (u/since-ms timer)]
+ (if (seq report)
+ (do
+ (analytics/inc! :metabase-search/index-reindex-ms duration)
+ (prometheus/observe! :metabase-search/index-reindex-duration-ms duration)
+ (doseq [[model cnt] report]
+ (analytics/inc! :metabase-search/index-reindexes {:model model} cnt))
+ (log/infof "Index initialized in %.0fms %s" duration (sort-by (comp - val) report))
+ report)
+ (log/info "Found existing search index, and using it.")))
+ (catch Exception e
+ (analytics/inc! :metabase-search/index-error)
+ (throw e))))))
(defn- reindex-logic! [opts]
(when (supports-index?)
- (try
- (log/info "Reindexing searchable entities")
- (let [timer (u/start-timer)
- report (reduce (partial merge-with max)
- nil
- (for [e (search.engine/active-engines)]
- (search.engine/reindex! e opts)))
- duration (u/since-ms timer)]
- (analytics/inc! :metabase-search/index-reindex-ms duration)
- (prometheus/observe! :metabase-search/index-reindex-duration-ms duration)
- (doseq [[model cnt] report]
- (analytics/inc! :metabase-search/index-reindexes {:model model} cnt))
- (log/infof "Done reindexing in %.0fms %s" duration (sort-by (comp - val) report))
- report)
- (catch Exception e
- (analytics/inc! :metabase-search/index-error)
- (throw e)))))
+ (lib-be/with-metadata-provider-cache
+ (try
+ (log/info "Reindexing searchable entities")
+ (let [timer (u/start-timer)
+ report (reduce (partial merge-with max)
+ nil
+ (for [e (search.engine/active-engines)]
+ (search.engine/reindex! e opts)))
+ duration (u/since-ms timer)]
+ (analytics/inc! :metabase-search/index-reindex-ms duration)
+ (prometheus/observe! :metabase-search/index-reindex-duration-ms duration)
+ (doseq [[model cnt] report]
+ (analytics/inc! :metabase-search/index-reindexes {:model model} cnt))
+ (log/infof "Done reindexing in %.0fms %s" duration (sort-by (comp - val) report))
+ report)
+ (catch Exception e
+ (analytics/inc! :metabase-search/index-error)
+ (throw e))))))
(defn reindex!
"Populate a new index, and make it active. Simultaneously updates the current index.
diff --git a/src/metabase/search/ingestion.clj b/src/metabase/search/ingestion.clj
index 3066b9c9990..355fd7a1b16 100644
--- a/src/metabase/search/ingestion.clj
+++ b/src/metabase/search/ingestion.clj
@@ -6,6 +6,7 @@
[metabase.analytics.core :as analytics]
[metabase.analytics.prometheus :as prometheus]
[metabase.app-db.core :as mdb]
+ [metabase.lib-be.core :as lib-be]
[metabase.search.engine :as search.engine]
[metabase.search.spec :as search.spec]
[metabase.util :as u]
@@ -270,23 +271,24 @@
(defn bulk-ingest!
"Process the given search model updates."
[updates]
- (if (seq (search.engine/active-engines))
- (let [documents (->> (for [[search-model where-clauses] (u/group-by first second updates)]
- (spec-index-reducible search-model (into [:or] (distinct where-clauses))))
- ;; init collection is only for clj-kondo, as we know that the list is non-empty
- (reduce u/rconcat [])
- query->documents)
- passed-documents (map extract-model-and-id updates)
- indexed-documents (map (juxt :model (comp str :id)) (into [] documents))
- ;; TODO: The list of documents to delete is not completely accurate.
- ;; We are attempting to figure it out based on the ids that are passed in to be indexed vs. the ids of the rows that were actually indexed.
- ;; This will not work for cases like indexed-entries with compound PKs,
- ;; but it's fine for now because that model doesn't have a where clause so never needs to be purged during an update.
- ;; Long-term, we should find a better approach to knowing what to purge.
- to-delete (remove (set indexed-documents) passed-documents)]
-
- (update! documents to-delete))
- {}))
+ (lib-be/with-metadata-provider-cache
+ (if (seq (search.engine/active-engines))
+ (let [documents (->> (for [[search-model where-clauses] (u/group-by first second updates)]
+ (spec-index-reducible search-model (into [:or] (distinct where-clauses))))
+ ;; init collection is only for clj-kondo, as we know that the list is non-empty
+ (reduce u/rconcat [])
+ query->documents)
+ passed-documents (map extract-model-and-id updates)
+ indexed-documents (map (juxt :model (comp str :id)) (into [] documents))
+ ;; TODO: The list of documents to delete is not completely accurate.
+ ;; We are attempting to figure it out based on the ids that are passed in to be indexed vs. the ids of the rows that were actually indexed.
+ ;; This will not work for cases like indexed-entries with compound PKs,
+ ;; but it's fine for now because that model doesn't have a where clause so never needs to be purged during an update.
+ ;; Long-term, we should find a better approach to knowing what to purge.
+ to-delete (remove (set indexed-documents) passed-documents)]
+
+ (update! documents to-delete))
+ {})))
(defn- track-queue-size! []
(analytics/set! :metabase-search/queue-size (.size queue)))
diff --git a/test/metabase/search/ingestion_test.clj b/test/metabase/search/ingestion_test.clj
index 0b6cfe5b3b1..2ad2348b9f2 100644
--- a/test/metabase/search/ingestion_test.clj
+++ b/test/metabase/search/ingestion_test.clj
@@ -1,6 +1,11 @@
(ns metabase.search.ingestion-test
(:require
+ [clojure.core.cache :as cache]
[clojure.test :refer :all]
+ [metabase.lib-be.metadata.jvm :as metadata.jvm]
+ [metabase.lib.metadata.protocols :as lib.metadata.protocols]
+ [metabase.search.core :as search]
+ [metabase.search.engine :as search.engine]
[metabase.search.ingestion :as search.ingestion]
[metabase.search.spec :as search.spec]
[metabase.test :as mt]))
@@ -106,3 +111,71 @@
:model/Dashboard _ {:name "Test Dashboard 2" :collection_id (:id coll)}]
(let [n (search.ingestion/search-items-count)]
(is (= 4 n))))))))
+
+(def ^:private fake-provider
+ (reify lib.metadata.protocols/MetadataProvider
+ (database [_this] nil)
+ (metadatas [_this _metadata-spec] nil)
+ (setting [_this _setting-name] nil)))
+
+(defn- counting-factory
+ "Returns a factory fn that counts calls in `counter-atom` and returns `fake-provider`."
+ [counter-atom]
+ (fn [_db-id]
+ (swap! counter-atom inc)
+ fake-provider))
+
+(defn- simulate-metadata-lookups
+ "Simulates what happens during indexing: the engine function calls
+ `application-database-metadata-provider` several times for the same database-id,
+ as it would when processing multiple cards from the same database."
+ []
+ ;; Three lookups for the same db — mimics processing multiple cards from one database.
+ (metadata.jvm/application-database-metadata-provider 1)
+ (metadata.jvm/application-database-metadata-provider 1)
+ (metadata.jvm/application-database-metadata-provider 1)
+ {"card" 3})
+
+(deftest bulk-ingest!-caches-metadata-providers-test
+ (testing "bulk-ingest! caches metadata providers so the factory is only called once per database-id"
+ (let [factory-calls (atom 0)]
+ (with-redefs [search.engine/active-engines (fn []
+ (simulate-metadata-lookups)
+ nil)
+ metadata.jvm/application-database-metadata-provider-factory (counting-factory factory-calls)]
+ (search.ingestion/bulk-ingest! [])
+ (is (= 1 @factory-calls)
+ "Factory should be called once, not once per metadata-provider lookup")))))
+
+(deftest bulk-ingest!-preserves-existing-cache-test
+ (testing "bulk-ingest! reuses an outer cache and does not rebuild providers already in it"
+ (let [factory-calls (atom 0)
+ existing-cache (atom (cache/basic-cache-factory {}))]
+ (with-redefs [search.engine/active-engines (fn []
+ (simulate-metadata-lookups)
+ nil)
+ metadata.jvm/application-database-metadata-provider-factory (counting-factory factory-calls)]
+ (binding [metadata.jvm/*metadata-provider-cache* existing-cache]
+ (metadata.jvm/application-database-metadata-provider 1)
+ (let [pre-calls @factory-calls]
+ (search.ingestion/bulk-ingest! [])
+ (is (= pre-calls @factory-calls)
+ "Factory should not be called again — the provider was already cached")))))))
+
+(deftest multiple-databases-cached-separately-test
+ (testing "Within a single indexing operation, each database-id gets its own cached provider"
+ (let [factory-calls (atom 0)]
+ (with-redefs [search.engine/active-engines (constantly [:search.engine/appdb])
+ search.engine/init! (fn [_engine _opts]
+ ;; Simulate cards from 3 different databases, 2 lookups each
+ (metadata.jvm/application-database-metadata-provider 1)
+ (metadata.jvm/application-database-metadata-provider 2)
+ (metadata.jvm/application-database-metadata-provider 3)
+ (metadata.jvm/application-database-metadata-provider 1)
+ (metadata.jvm/application-database-metadata-provider 2)
+ (metadata.jvm/application-database-metadata-provider 3)
+ {"card" 6})
+ metadata.jvm/application-database-metadata-provider-factory (counting-factory factory-calls)]
+ (search/init-index!)
+ (is (= 3 @factory-calls)
+ "Factory should be called once per unique database-id, not once per lookup")))))