Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Bgrabow/monitor memoization cache size (#75967)

Ben Grabow1d050aec9c8a
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index dfd626cd0d9..6d6de8f66bd 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -357,6 +357,7 @@
{:team "UX West"
:api #{metabase.audit-app.core
metabase.audit-app.init}
+ :friends #{enterprise/memoize-monitor}
:uses #{api
app-db
events
@@ -3222,6 +3223,20 @@
premium-features}
:model-imports #{:model/Card :model/Collection :model/Table}}
+ enterprise/memoize-monitor
+ {:team "Graphy"
+ :api #{metabase-enterprise.memoize-monitor.init}
+ :uses #{analytics
+ analytics-interface
+ audit-app
+ collections
+ driver
+ models
+ util
+ warehouse-schema
+ warehouses
+ enterprise/serialization}}
+
enterprise/metabot
{:team "Metabot"
:api #{metabase-enterprise.metabot.api
@@ -3458,6 +3473,7 @@
metabase-enterprise.serialization.init
metabase-enterprise.serialization.schema
metabase-enterprise.serialization.v2.backfill-ids}
+ :friends #{enterprise/memoize-monitor}
:uses #{analytics
api
app-db
diff --git a/enterprise/backend/src/metabase_enterprise/core/init.clj b/enterprise/backend/src/metabase_enterprise/core/init.clj
index 227749a7ad8..a93027616b4 100644
--- a/enterprise/backend/src/metabase_enterprise/core/init.clj
+++ b/enterprise/backend/src/metabase_enterprise/core/init.clj
@@ -16,6 +16,7 @@
[metabase-enterprise.database-replication.init]
[metabase-enterprise.dependencies.init]
[metabase-enterprise.gsheets.init]
+ [metabase-enterprise.memoize-monitor.init]
[metabase-enterprise.metabot.init]
[metabase-enterprise.remote-sync.init]
[metabase-enterprise.scim.init]
diff --git a/enterprise/backend/src/metabase_enterprise/memoize_monitor/init.clj b/enterprise/backend/src/metabase_enterprise/memoize_monitor/init.clj
new file mode 100644
index 00000000000..a7845c925c2
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/memoize_monitor/init.clj
@@ -0,0 +1,76 @@
+(ns metabase-enterprise.memoize-monitor.init
+ "Reports the size of a curated set of in-memory memoization caches as telemetry.
+
+ Some memoization caches in the app are keyed on argument spaces that can grow large or unbounded (field IDs, table
+ IDs, decrypted-secret ciphertexts, per-request collection-visibility configs, serialization paths, ...). Rather than
+ instrument each cache at its definition site, this namespace references the vars that hold those caches directly and
+ reads each cache's size on a periodic basis via a Prometheus `pull-collector`. For each monitored cache we emit:
+
+ - `:metabase-memoize/cache-size` -- number of entries
+
+ This lives in enterprise because the monitored caches span both OSS and enterprise modules
+ ([[metabase-enterprise.serialization.dump]]); an OSS namespace cannot depend on enterprise code. OSS-only builds get
+ no memoize-cache telemetry.
+
+ Only caches built with [[clojure.core.memoize]] (`memo`, `ttl`, and
+ [[metabase.app-db.core/memoize-for-application-db]], which is built on `memo`) can be measured, because their
+ backing cache is reachable through the memoized function's metadata."
+ (:require
+ [metabase-enterprise.serialization.dump :as serialization.dump]
+ [metabase.analytics-interface.core :as analytics.interface]
+ [metabase.analytics.core :as analytics]
+ [metabase.audit-app.impl :as audit-app.impl]
+ [metabase.collections.models.collection :as collection]
+ [metabase.driver.util :as driver.u]
+ [metabase.models.interface :as mi]
+ [metabase.util.log :as log]
+ [metabase.warehouse-schema.models.field :as schema.field]
+ [metabase.warehouses.models.database :as database])
+ (:import
+ (clojure.core.memoize PluggableMemoization)))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private monitored-cache-vars
+ "The vars holding the memoization caches whose sizes we export. Each must hold a [[clojure.core.memoize]]-backed
+ function (see the namespace docstring)."
+ [#'schema.field/field-id->table-id
+ #'database/table-id->database-id
+ #'database/db-id->router-db-id
+ #'audit-app.impl/memoized-select-audit-entity*
+ #'driver.u/memoized-supports?*
+ #'driver.u/memoized-features*
+ #'mi/cached-encrypted-json-out
+ #'collection/can-access-root-collection?
+ #'collection/visible-collection-ids*
+ #'serialization.dump/serialization-sorted-map])
+
+(defn- cache-object
+ "The backing cache of a [[clojure.core.memoize]]-memoized function `f`, or `nil` if it isn't reachable. The returned
+ object is a `clojure.core.cache` cache that is both `count`able and measurable."
+ [f]
+ (when-let [pm (some-> f meta :clojure.core.memoize/cache deref)]
+ (when (instance? PluggableMemoization pm)
+ (.cache ^PluggableMemoization pm))))
+
+(defn- memoization-cache-stats
+ "Map of `{:cache name, :entries n}` given a var holding a memoized function with a reachable backing cache, or `nil`
+ otherwise."
+ [cache-var]
+ (try
+ (when-let [cache (cache-object @cache-var)]
+ (let [entries (count cache)]
+ {:cache (str (symbol cache-var))
+ :entries entries}))
+ (catch Exception e
+ (log/warn e "Error measuring memoization cache size" {:cache (str (symbol cache-var))}))))
+
+(defn- all-cache-stats
+ []
+ (keep memoization-cache-stats monitored-cache-vars))
+
+(defmethod analytics/pull-collector ::memoize-cache-sizes [_]
+ {:min-interval-s (* 10 60)
+ :f (fn []
+ (doseq [{:keys [cache entries]} (all-cache-stats)]
+ (analytics.interface/set-gauge! :metabase-memoize/cache-size {:cache cache} entries)))})
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/dump.clj b/enterprise/backend/src/metabase_enterprise/serialization/dump.clj
index 02404dc1eae..c0012e8daf2 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/dump.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/dump.clj
@@ -1,6 +1,7 @@
(ns metabase-enterprise.serialization.dump
"Serialize entities into a directory structure of YAMLs."
(:require
+ [clojure.core.memoize :as memoize]
[clojure.edn :as edn]
[clojure.java.io :as io]
[metabase.util.yaml :as yaml]))
@@ -23,7 +24,7 @@
(compare (getter k1) (getter k2)))))
(sorted-map)))
-(def ^:private serialization-sorted-map (memoize serialization-sorted-map*))
+(def ^:private serialization-sorted-map (memoize/memo serialization-sorted-map*))
(defn serialization-deep-sort
"Provide a deterministic sort for maps before serialization."
diff --git a/enterprise/backend/test/metabase_enterprise/memoize_monitor/init_test.clj b/enterprise/backend/test/metabase_enterprise/memoize_monitor/init_test.clj
new file mode 100644
index 00000000000..2f4255f7a7b
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/memoize_monitor/init_test.clj
@@ -0,0 +1,29 @@
+(ns metabase-enterprise.memoize-monitor.init-test
+ (:require
+ [clojure.core.memoize :as memoize]
+ [clojure.test :refer :all]
+ [metabase-enterprise.memoize-monitor.init :as memoize-monitor]))
+
+(set! *warn-on-reflection* true)
+
+(deftest cache-object-test
+ (testing "returns the backing cache of a clojure.core.memoize function (countable + measurable)"
+ (let [f (memoize/memo (fn [x] (* x x)))]
+ (f 1) (f 2) (f 2)
+ (is (= 2 (count (#'memoize-monitor/cache-object f))))))
+ (testing "returns nil for a function whose backing cache isn't reachable"
+ (is (nil? (#'memoize-monitor/cache-object (fn [x] x))))
+ (is (nil? (#'memoize-monitor/cache-object (clojure.core/memoize (fn [x] x)))))))
+
+(deftest cache-stats-entries-test
+ (let [stats (#'memoize-monitor/all-cache-stats)
+ by-cache (into {} (map (juxt :cache identity)) stats)]
+ (testing "one stat per monitored cache, keyed by the var's fully-qualified symbol"
+ (is (= (count @#'memoize-monitor/monitored-cache-vars)
+ (count stats)))
+ (is (contains? by-cache "metabase.warehouse-schema.models.field/field-id->table-id"))
+ (is (contains? by-cache "metabase-enterprise.serialization.dump/serialization-sorted-map")))
+ (testing "entry counts are always reported"
+ (doseq [{:keys [cache entries]} stats]
+ (is (string? cache))
+ (is (and (integer? entries) (not (neg? entries))) (str cache " entries"))))))
diff --git a/src/metabase/analytics/prometheus.clj b/src/metabase/analytics/prometheus.clj
index 0f1832543b8..aa2784f4ead 100644
--- a/src/metabase/analytics/prometheus.clj
+++ b/src/metabase/analytics/prometheus.clj
@@ -773,7 +773,10 @@
;; metaplow analytics metrics
(prometheus/counter :metabase-metaplow/errors
{:description "Metaplow event pipeline errors by stage."
- :labels [:stage]})])
+ :labels [:stage]})
+ (prometheus/gauge :metabase-memoize/cache-size
+ {:description "Number of entries currently held in a monitored in-memory memoization cache."
+ :labels [:cache]})])
(defn- quartz-collectors
[]
diff --git a/src/metabase/app_db/core.clj b/src/metabase/app_db/core.clj
index f55ab7dca89..cf38535a856 100644
--- a/src/metabase/app_db/core.clj
+++ b/src/metabase/app_db/core.clj
@@ -9,6 +9,7 @@
Namespaces outside of src/metabase/app_db/ should not use any metabase.app-db.* namespace but use this api namespace."
(:refer-clojure :exclude [compile])
(:require
+ [clojure.core.memoize :as memoize]
[metabase.app-db.connection :as mdb.connection]
[metabase.app-db.connection-pool-setup :as mdb.connection-pool-setup]
[metabase.app-db.data-source :as mdb.data-source]
@@ -153,10 +154,10 @@
`:clojure.core.memoize/args-fn` instead; see [[metabase.driver.util/database->driver*]] for an example of how to do
this."
[f]
- (let [f* (memoize (fn [_application-db-id & args]
- (apply f args)))]
- (fn [& args]
- (apply f* (unique-identifier) args))))
+ (memoize/memo
+ (vary-meta
+ (fn [& args] (apply f args))
+ assoc ::memoize/args-fn (fn [args] (cons (unique-identifier) args)))))
(defn increment-app-db-unique-indentifier!
"Increment the [[unique-identifier]] for the Metabase application DB. This effectively flushes all caches using it as
diff --git a/test/metabase/app_db/core_test.clj b/test/metabase/app_db/core_test.clj
new file mode 100644
index 00000000000..4c73d03f553
--- /dev/null
+++ b/test/metabase/app_db/core_test.clj
@@ -0,0 +1,23 @@
+(ns metabase.app-db.core-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase.app-db.connection :as mdb.connection]
+ [metabase.app-db.core :as mdb]))
+
+(deftest memoize-for-application-db-test
+ (let [calls (atom 0)
+ f (mdb/memoize-for-application-db
+ (fn [x]
+ (swap! calls inc)
+ (inc x)))]
+ (testing "memoizes: the underlying fn runs once per distinct argument"
+ (is (= 2 (f 1)))
+ (is (= 2 (f 1)))
+ (is (= 3 (f 2)))
+ (is (= 2 @calls)))
+ (testing "cache is keyed by application DB, so a different app DB recomputes"
+ (mdb/with-application-db (assoc mdb.connection/*application-db* :id Integer/MAX_VALUE)
+ (is (= 2 (f 1))))
+ (is (= 3 @calls)))
+ (testing "the cache is introspectable for telemetry (carries clojure.core.memoize cache metadata)"
+ (is (some? (:clojure.core.memoize/cache (meta f)))))))