Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Switch appdb-index-size to be a real db check (#75284)

Nathan Voxlandedaf5a64f732
diff --git a/src/metabase/analytics/core.clj b/src/metabase/analytics/core.clj
index 2e3949dc307..1fc7818293b 100644
--- a/src/metabase/analytics/core.clj
+++ b/src/metabase/analytics/core.clj
@@ -33,6 +33,7 @@
initial-value
connection-pool-info
observe-initial-values
+ pull-collector
setup!
shutdown!]
[metabase.analytics.quartz
diff --git a/src/metabase/analytics/prometheus.clj b/src/metabase/analytics/prometheus.clj
index 46236331516..0f1832543b8 100644
--- a/src/metabase/analytics/prometheus.clj
+++ b/src/metabase/analytics/prometheus.clj
@@ -170,6 +170,46 @@
([] (collect-metrics))
([_sampleNameFilter] (collect-metrics))))))))
+(defmulti pull-collector
+ "Declare a collect-time metric refresher. Each implementation returns a map of:
+
+ `:min-interval-s` -- run no more often than once every this many seconds
+ `:f` -- a 0-arg function that refreshes one or more *declared* Prometheus metrics by making
+ whatever update calls it wants (e.g. [[metabase.analytics-interface.core/set-gauge!]]
+ or [[inc!]], for any number of metrics)
+
+ The dispatch value is any unique keyword identifying the collector. Implementations are run by
+ [[product-pull-collectors]] on each scrape (throttled by min-interval-s)."
+ {:arglists '([id])}
+ identity)
+
+(defonce ^:private pull-collector-last-runs
+ ;; id -> epoch-ms the collector's function last ran, used to enforce its :min-interval-s
+ (atom {}))
+
+(def ^:private product-pull-collectors
+ "Registered collector that runs the [[pull-collector]] implementations at scrape time, each throttled to
+ its own `:min-interval-s`. A throwing function is skipped without failing the scrape."
+ (letfn [(run-all! []
+ (let [now (System/currentTimeMillis)]
+ (doseq [id (keys (methods pull-collector))]
+ (try
+ (let [{:keys [min-interval-s f]} (pull-collector id)]
+ (when (>= (- now (get @pull-collector-last-runs id 0)) (long (* 1000 min-interval-s)))
+ (swap! pull-collector-last-runs assoc id now)
+ (f)))
+ (catch Throwable e
+ (log/warn e "Error running pull collector" id)))))
+ [])]
+ (delay
+ (collector/named
+ {:name "metabase_application_pull"
+ :namespace "metabase"}
+ (proxy [Collector] []
+ (collect
+ ([] (run-all!))
+ ([_sampleNameFilter] (run-all!))))))))
+
(defn- jvm-collectors
"JVM collectors. Essentially duplicating [[iapetos.collector.jvm]] namespace so we can set our own namespaces rather
than \"iapetos_internal\""
@@ -318,7 +358,7 @@
;; 1ms -> 10minutes
:buckets [1 500 1000 5000 10000 30000 60000 120000 300000 600000]})
(prometheus/gauge :metabase-search/appdb-index-size
- {:description "Number of rows in the active appdb index table."})
+ {:description "Estimated number of rows in this instance's active appdb search index table."})
(prometheus/gauge :metabase-search/semantic-index-size
{:description "Number of rows in the active semantic index table."})
(prometheus/gauge :metabase-search/semantic-dlq-size
@@ -784,7 +824,8 @@
(collector.ring/initialize registry)
(concat (jvm-collectors)
(jetty-collectors)
- [@c3p0-collector]
+ [@c3p0-collector
+ @product-pull-collectors]
(product-collectors)
(quartz-collectors)))]
(when @jvm-hiccup-thread (@jvm-hiccup-thread))
diff --git a/src/metabase/search/appdb/index.clj b/src/metabase/search/appdb/index.clj
index cd4f7889e27..128bd4fed5e 100644
--- a/src/metabase/search/appdb/index.clj
+++ b/src/metabase/search/appdb/index.clj
@@ -104,7 +104,10 @@
(cond-> (name kw)
(= :h2 (mdb/db-type)) u/upper-case-en))
-(defn- exists? [table]
+(defn exists?
+ "Whether the given index `table` actually exists in the appdb (the tracked active/pending table can be
+ briefly stale relative to what has been dropped)."
+ [table]
(when table
(t2/exists? :information_schema.tables :table_name (table-name table))))
@@ -353,12 +356,7 @@
(u/prog1 (->> entries (map :model) frequencies)
(when reindexing?
(t2/query ["commit"]))
- (log/trace "indexed documents for " <>)
- (when active-updated
- (try
- (analytics/set-gauge! :metabase-search/appdb-index-size (t2/count (name active-updated)))
- (catch Exception e
- (log/warnf e "Unable to measure active search index size (%s)" active-updated))))))))]
+ (log/trace "indexed documents for " <>)))))]
(if reindexing?
;; New connection used for performing the updates which commit periodically without impacting any outer transactions.
(t2/with-connection [_conn (mdb/data-source)]
@@ -380,23 +378,14 @@
(defmethod search.engine/delete! :search.engine/appdb [_engine search-model ids]
(when (seq ids)
- (u/prog1 (->> [(active-table) (pending-table)]
- (keep (fn [table-name]
- (when table-name
- {search-model (try (t2/delete! table-name :model search-model :model_id [:in (set ids)])
- ;; Race conditions with table being deleted, especially in tests.
- (catch Exception e (if (table-not-found-exception? e) 0 (throw e))))})))
- (apply merge-with +)
- (into {}))
- (when (active-table)
- (try
- (analytics/set-gauge! :metabase-search/appdb-index-size (:count (t2/query-one {:select [[:%count.* :count]]
- :from [(active-table)]
- :limit 1})))
- (catch Exception e
- ;; No point tracking the size of the newer index table, since we won't have modified it.
- (when-not (table-not-found-exception? e)
- (throw e))))))))
+ (->> [(active-table) (pending-table)]
+ (keep (fn [table-name]
+ (when table-name
+ {search-model (try (t2/delete! table-name :model search-model :model_id [:in (set ids)])
+ ;; Race conditions with table being deleted, especially in tests.
+ (catch Exception e (if (table-not-found-exception? e) 0 (throw e))))})))
+ (apply merge-with +)
+ (into {}))))
(defn when-index-created
"Return creation time of the active index, or nil if there is none."
diff --git a/src/metabase/search/appdb/metrics.clj b/src/metabase/search/appdb/metrics.clj
new file mode 100644
index 00000000000..e84b8b717e6
--- /dev/null
+++ b/src/metabase/search/appdb/metrics.clj
@@ -0,0 +1,24 @@
+(ns metabase.search.appdb.metrics
+ "Prometheus instrumentation for the appdb search engine."
+ (:require
+ [metabase.analytics-interface.core :as analytics.interface]
+ [metabase.analytics.core :as analytics]
+ [metabase.search.appdb.index :as search.index]
+ [metabase.search.appdb.specialization.api :as specialization]
+ [metabase.search.engine :as search.engine]))
+
+(defn- active-index-size
+ []
+ (when-let [table (search.index/active-table)]
+ (when (search.index/exists? table)
+ (specialization/index-size-estimate table))))
+
+;; Keep the :metabase-search/appdb-index-size product gauge up to date (per-instance). Skips the work entirely
+;; unless appdb is a supported engine here, so this is safe to leave registered regardless of which search
+;; engine is in use. The estimate is cheap (no full scan), so it's refreshed roughly once a minute.
+(defmethod analytics/pull-collector ::index-size [_]
+ {:min-interval-s 60
+ :f (fn []
+ (when (search.engine/supported-engine? :search.engine/appdb)
+ (when-let [n (active-index-size)]
+ (analytics.interface/set-gauge! :metabase-search/appdb-index-size n))))})
diff --git a/src/metabase/search/appdb/specialization/api.clj b/src/metabase/search/appdb/specialization/api.clj
index e70a505d1f8..9369804b745 100644
--- a/src/metabase/search/appdb/specialization/api.clj
+++ b/src/metabase/search/appdb/specialization/api.clj
@@ -38,3 +38,9 @@
"Calculate the given p-value of view counts for each search model"
{:arglists '([index-table p-value])}
db-type)
+
+(defmulti index-size-estimate
+ "An estimated row count for the index `table-name`, ideally without scanning the whole table (used for the
+ `metabase_search_appdb_index_size` metric, which is scraped frequently)."
+ {:arglists '([table-name])}
+ db-type)
diff --git a/src/metabase/search/appdb/specialization/h2.clj b/src/metabase/search/appdb/specialization/h2.clj
index e8a0facba90..9bb52b7f72b 100644
--- a/src/metabase/search/appdb/specialization/h2.clj
+++ b/src/metabase/search/appdb/specialization/h2.clj
@@ -61,3 +61,7 @@
{:select [:search_index.model [[:* [:inline (math/pow p-value 10)] [:max :view_count]] :vcp]]
:from [[index-table :search_index]]
:group-by [:search_index.model]})
+
+(defmethod specialization/index-size-estimate :h2
+ [table-name]
+ (t2/count table-name))
diff --git a/src/metabase/search/appdb/specialization/postgres.clj b/src/metabase/search/appdb/specialization/postgres.clj
index fa9a0e2e216..116579cf56b 100644
--- a/src/metabase/search/appdb/specialization/postgres.clj
+++ b/src/metabase/search/appdb/specialization/postgres.clj
@@ -98,3 +98,14 @@
:from [[index-table :search_index]]
:group-by [:search_index.model]
:having [:is-not expr nil]}))
+
+(defmethod specialization/index-size-estimate :postgres
+ [table-name]
+ ;; Use the planner's row estimate (pg_class.reltuples) instead of a full count(*). reltuples/relpages are
+ ;; only populated by ANALYZE/VACUUM, so right after a rebuild the estimate may be stale — return nil in
+ ;; that window and let the caller skip the metric rather than doing a full table scan.
+ (let [{:keys [reltuples relpages]} (t2/query-one {:select [:reltuples :relpages]
+ :from [:pg_class]
+ :where [:= :oid [:to_regclass (name table-name)]]})]
+ (when (and relpages (pos? relpages) reltuples (nat-int? (long reltuples)))
+ (long reltuples))))
diff --git a/src/metabase/search/init.clj b/src/metabase/search/init.clj
index 43b66a66d0a..0979fcc541f 100644
--- a/src/metabase/search/init.clj
+++ b/src/metabase/search/init.clj
@@ -2,6 +2,7 @@
"This is loaded for side effects on system launch."
(:require
[metabase.search.appdb.core]
+ [metabase.search.appdb.metrics]
[metabase.search.in-place.legacy]
[metabase.search.models]
[metabase.search.semantic.core]
diff --git a/test/metabase/analytics/prometheus_test.clj b/test/metabase/analytics/prometheus_test.clj
index 26e248a744c..ac7118ab845 100644
--- a/test/metabase/analytics/prometheus_test.clj
+++ b/test/metabase/analytics/prometheus_test.clj
@@ -193,6 +193,24 @@
(prometheus/dec! :metabase-search/engine-active {:engine :default} 1)
(is (approx= -1 (mt/metric-value system :metabase-search/engine-active {:engine :default}))))))
+(deftest pull-collector-test
+ (testing "a pull collector implementation runs at scrape time and can update one or more declared metrics"
+ (mt/with-prometheus-system! [_ system]
+ (let [refresher (registry/get (:registry system)
+ {:name "metabase_application_pull" :namespace "metabase"} nil)]
+ (try
+ ;; the function makes whatever metric updates it wants -- here it sets the (declared)
+ ;; :metabase-search/appdb-index-size gauge
+ (defmethod prometheus/pull-collector ::test [_]
+ {:min-interval-s 0
+ :f (fn []
+ (prometheus/set! :metabase-search/appdb-index-size 7))})
+ (.collect ^Collector refresher) ; runs the registered functions
+ (is (approx= 7 (mt/metric-value system :metabase-search/appdb-index-size)))
+ (finally
+ (remove-method prometheus/pull-collector ::test)
+ (swap! @#'prometheus/pull-collector-last-runs dissoc ::test)))))))
+
(deftest search-engine-metrics-test
(let [metrics (#'prometheus/initial-labelled-metric-values)
engine->value (fn [metric] (u/index-by (comp :engine :labels) :value (filter (comp #{metric} :metric) metrics)))
diff --git a/test/metabase/search/appdb/metrics_test.clj b/test/metabase/search/appdb/metrics_test.clj
new file mode 100644
index 00000000000..d5f503c9c37
--- /dev/null
+++ b/test/metabase/search/appdb/metrics_test.clj
@@ -0,0 +1,39 @@
+(ns metabase.search.appdb.metrics-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase.app-db.core :as mdb]
+ [metabase.search.appdb.index :as search.index]
+ [metabase.search.appdb.metrics :as metrics]
+ [metabase.search.engine :as search.engine]
+ [metabase.search.ingestion :as search.ingestion]
+ [metabase.search.test-util :as search.tu]
+ [metabase.test :as mt]
+ [metabase.test.fixtures :as fixtures]
+ [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(use-fixtures :once (fixtures/initialize :db :test-users))
+
+(deftest active-index-size-test
+ (testing "estimated row count of the active index, for the metabase_search_appdb_index_size metric -- see #75064"
+ (search.tu/with-temp-index-table
+ (binding [search.ingestion/*force-sync* true]
+ (mt/dataset test-data
+ (mt/with-temp [:model/User {} (when-not (t2/exists? :model/User 1) {:id 1})
+ :model/Collection {col-id :id} {:name "Collection"}
+ :model/Card {} {:name "Customer Satisfaction" :collection_id col-id}
+ :model/Card {} {:name "Projected Revenue" :collection_id col-id}]
+ (search.engine/reindex! :search.engine/appdb {:in-place? true})
+ (let [exact (t2/count (search.index/active-table))]
+ (is (pos? exact))
+ (if (= :postgres (mdb/db-type))
+ (testing "uses the pg_class estimate (no full count(*)) after ANALYZE"
+ (t2/query (str "ANALYZE " (name (search.index/active-table))))
+ (is (= exact (#'metrics/active-index-size))))
+ (testing "returns an exact count on H2"
+ (is (= exact (#'metrics/active-index-size)))))))))))
+ (testing "returns nil when this instance has no active index to serve from"
+ (search.tu/with-temp-index-table
+ (reset! @#'search.index/*indexes* {:active nil :pending nil})
+ (is (nil? (#'metrics/active-index-size))))))