The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Search Indexing: Cache card schema upgrades (#68912)
Nathan Voxland6e4d96a68f57
diff --git a/src/metabase/search/core.clj b/src/metabase/search/core.cljindex 1df3c034744..2623ebd8a55 100644--- a/src/metabase/search/core.clj+++ b/src/metabase/search/core.clj@@ -3,6 +3,7 @@(:require[metabase.analytics.core :as analytics][metabase.analytics.prometheus :as prometheus]+ [metabase.lib-be.core :as lib-be][metabase.search.config :as search.config][metabase.search.engine :as search.engine][metabase.search.impl :as search.impl]@@ -86,46 +87,48 @@[& {:as opts}](when (supports-index?)(log/info "Initializing search indexes")- ;; If there are multiple indexes, return the peak inserted for each type. In practice, they should all be the same.- (try- (let [timer (u/start-timer)- report (reduce (partial merge-with max)- nil- (for [e (search.engine/active-engines)]- (search.engine/init! e opts)))- duration (u/since-ms timer)]- (if (seq report)- (do- (analytics/inc! :metabase-search/index-reindex-ms duration)- (prometheus/observe! :metabase-search/index-reindex-duration-ms duration)- (doseq [[model cnt] report]- (analytics/inc! :metabase-search/index-reindexes {:model model} cnt))- (log/infof "Index initialized in %.0fms %s" duration (sort-by (comp - val) report))- report)- (log/info "Found existing search index, and using it.")))- (catch Exception e- (analytics/inc! :metabase-search/index-error)- (throw e)))))+ (lib-be/with-metadata-provider-cache+ ;; If there are multiple indexes, return the peak inserted for each type. In practice, they should all be the same.+ (try+ (let [timer (u/start-timer)+ report (reduce (partial merge-with max)+ nil+ (for [e (search.engine/active-engines)]+ (search.engine/init! e opts)))+ duration (u/since-ms timer)]+ (if (seq report)+ (do+ (analytics/inc! :metabase-search/index-reindex-ms duration)+ (prometheus/observe! :metabase-search/index-reindex-duration-ms duration)+ (doseq [[model cnt] report]+ (analytics/inc! :metabase-search/index-reindexes {:model model} cnt))+ (log/infof "Index initialized in %.0fms %s" duration (sort-by (comp - val) report))+ report)+ (log/info "Found existing search index, and using it.")))+ (catch Exception e+ (analytics/inc! :metabase-search/index-error)+ (throw e))))))(defn- reindex-logic! [opts](when (supports-index?)- (try- (log/info "Reindexing searchable entities")- (let [timer (u/start-timer)- report (reduce (partial merge-with max)- nil- (for [e (search.engine/active-engines)]- (search.engine/reindex! e opts)))- duration (u/since-ms timer)]- (analytics/inc! :metabase-search/index-reindex-ms duration)- (prometheus/observe! :metabase-search/index-reindex-duration-ms duration)- (doseq [[model cnt] report]- (analytics/inc! :metabase-search/index-reindexes {:model model} cnt))- (log/infof "Done reindexing in %.0fms %s" duration (sort-by (comp - val) report))- report)- (catch Exception e- (analytics/inc! :metabase-search/index-error)- (throw e)))))+ (lib-be/with-metadata-provider-cache+ (try+ (log/info "Reindexing searchable entities")+ (let [timer (u/start-timer)+ report (reduce (partial merge-with max)+ nil+ (for [e (search.engine/active-engines)]+ (search.engine/reindex! e opts)))+ duration (u/since-ms timer)]+ (analytics/inc! :metabase-search/index-reindex-ms duration)+ (prometheus/observe! :metabase-search/index-reindex-duration-ms duration)+ (doseq [[model cnt] report]+ (analytics/inc! :metabase-search/index-reindexes {:model model} cnt))+ (log/infof "Done reindexing in %.0fms %s" duration (sort-by (comp - val) report))+ report)+ (catch Exception e+ (analytics/inc! :metabase-search/index-error)+ (throw e))))))(defn reindex!"Populate a new index, and make it active. Simultaneously updates the current index.diff --git a/src/metabase/search/ingestion.clj b/src/metabase/search/ingestion.cljindex 3066b9c9990..355fd7a1b16 100644--- a/src/metabase/search/ingestion.clj+++ b/src/metabase/search/ingestion.clj@@ -6,6 +6,7 @@[metabase.analytics.core :as analytics][metabase.analytics.prometheus :as prometheus][metabase.app-db.core :as mdb]+ [metabase.lib-be.core :as lib-be][metabase.search.engine :as search.engine][metabase.search.spec :as search.spec][metabase.util :as u]@@ -270,23 +271,24 @@(defn bulk-ingest!"Process the given search model updates."[updates]- (if (seq (search.engine/active-engines))- (let [documents (->> (for [[search-model where-clauses] (u/group-by first second updates)]- (spec-index-reducible search-model (into [:or] (distinct where-clauses))))- ;; init collection is only for clj-kondo, as we know that the list is non-empty- (reduce u/rconcat [])- query->documents)- passed-documents (map extract-model-and-id updates)- indexed-documents (map (juxt :model (comp str :id)) (into [] documents))- ;; TODO: The list of documents to delete is not completely accurate.- ;; We are attempting to figure it out based on the ids that are passed in to be indexed vs. the ids of the rows that were actually indexed.- ;; This will not work for cases like indexed-entries with compound PKs,- ;; but it's fine for now because that model doesn't have a where clause so never needs to be purged during an update.- ;; Long-term, we should find a better approach to knowing what to purge.- to-delete (remove (set indexed-documents) passed-documents)]-- (update! documents to-delete))- {}))+ (lib-be/with-metadata-provider-cache+ (if (seq (search.engine/active-engines))+ (let [documents (->> (for [[search-model where-clauses] (u/group-by first second updates)]+ (spec-index-reducible search-model (into [:or] (distinct where-clauses))))+ ;; init collection is only for clj-kondo, as we know that the list is non-empty+ (reduce u/rconcat [])+ query->documents)+ passed-documents (map extract-model-and-id updates)+ indexed-documents (map (juxt :model (comp str :id)) (into [] documents))+ ;; TODO: The list of documents to delete is not completely accurate.+ ;; We are attempting to figure it out based on the ids that are passed in to be indexed vs. the ids of the rows that were actually indexed.+ ;; This will not work for cases like indexed-entries with compound PKs,+ ;; but it's fine for now because that model doesn't have a where clause so never needs to be purged during an update.+ ;; Long-term, we should find a better approach to knowing what to purge.+ to-delete (remove (set indexed-documents) passed-documents)]++ (update! documents to-delete))+ {})))(defn- track-queue-size! [](analytics/set! :metabase-search/queue-size (.size queue)))diff --git a/test/metabase/search/ingestion_test.clj b/test/metabase/search/ingestion_test.cljindex 0b6cfe5b3b1..2ad2348b9f2 100644--- a/test/metabase/search/ingestion_test.clj+++ b/test/metabase/search/ingestion_test.clj@@ -1,6 +1,11 @@(ns metabase.search.ingestion-test(:require+ [clojure.core.cache :as cache][clojure.test :refer :all]+ [metabase.lib-be.metadata.jvm :as metadata.jvm]+ [metabase.lib.metadata.protocols :as lib.metadata.protocols]+ [metabase.search.core :as search]+ [metabase.search.engine :as search.engine][metabase.search.ingestion :as search.ingestion][metabase.search.spec :as search.spec][metabase.test :as mt]))@@ -106,3 +111,71 @@:model/Dashboard _ {:name "Test Dashboard 2" :collection_id (:id coll)}](let [n (search.ingestion/search-items-count)](is (= 4 n))))))))++(def ^:private fake-provider+ (reify lib.metadata.protocols/MetadataProvider+ (database [_this] nil)+ (metadatas [_this _metadata-spec] nil)+ (setting [_this _setting-name] nil)))++(defn- counting-factory+ "Returns a factory fn that counts calls in `counter-atom` and returns `fake-provider`."+ [counter-atom]+ (fn [_db-id]+ (swap! counter-atom inc)+ fake-provider))++(defn- simulate-metadata-lookups+ "Simulates what happens during indexing: the engine function calls+ `application-database-metadata-provider` several times for the same database-id,+ as it would when processing multiple cards from the same database."+ []+ ;; Three lookups for the same db — mimics processing multiple cards from one database.+ (metadata.jvm/application-database-metadata-provider 1)+ (metadata.jvm/application-database-metadata-provider 1)+ (metadata.jvm/application-database-metadata-provider 1)+ {"card" 3})++(deftest bulk-ingest!-caches-metadata-providers-test+ (testing "bulk-ingest! caches metadata providers so the factory is only called once per database-id"+ (let [factory-calls (atom 0)]+ (with-redefs [search.engine/active-engines (fn []+ (simulate-metadata-lookups)+ nil)+ metadata.jvm/application-database-metadata-provider-factory (counting-factory factory-calls)]+ (search.ingestion/bulk-ingest! [])+ (is (= 1 @factory-calls)+ "Factory should be called once, not once per metadata-provider lookup")))))++(deftest bulk-ingest!-preserves-existing-cache-test+ (testing "bulk-ingest! reuses an outer cache and does not rebuild providers already in it"+ (let [factory-calls (atom 0)+ existing-cache (atom (cache/basic-cache-factory {}))]+ (with-redefs [search.engine/active-engines (fn []+ (simulate-metadata-lookups)+ nil)+ metadata.jvm/application-database-metadata-provider-factory (counting-factory factory-calls)]+ (binding [metadata.jvm/*metadata-provider-cache* existing-cache]+ (metadata.jvm/application-database-metadata-provider 1)+ (let [pre-calls @factory-calls]+ (search.ingestion/bulk-ingest! [])+ (is (= pre-calls @factory-calls)+ "Factory should not be called again — the provider was already cached")))))))++(deftest multiple-databases-cached-separately-test+ (testing "Within a single indexing operation, each database-id gets its own cached provider"+ (let [factory-calls (atom 0)]+ (with-redefs [search.engine/active-engines (constantly [:search.engine/appdb])+ search.engine/init! (fn [_engine _opts]+ ;; Simulate cards from 3 different databases, 2 lookups each+ (metadata.jvm/application-database-metadata-provider 1)+ (metadata.jvm/application-database-metadata-provider 2)+ (metadata.jvm/application-database-metadata-provider 3)+ (metadata.jvm/application-database-metadata-provider 1)+ (metadata.jvm/application-database-metadata-provider 2)+ (metadata.jvm/application-database-metadata-provider 3)+ {"card" 6})+ metadata.jvm/application-database-metadata-provider-factory (counting-factory factory-calls)]+ (search/init-index!)+ (is (= 3 @factory-calls)+ "Factory should be called once per unique database-id, not once per lookup")))))