The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Bgrabow/monitor memoization cache size (#75967)
Ben Grabow1d050aec9c8a
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex dfd626cd0d9..6d6de8f66bd 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -357,6 +357,7 @@{:team "UX West":api #{metabase.audit-app.coremetabase.audit-app.init}+ :friends #{enterprise/memoize-monitor}:uses #{apiapp-dbevents@@ -3222,6 +3223,20 @@premium-features}:model-imports #{:model/Card :model/Collection :model/Table}}+ enterprise/memoize-monitor+ {:team "Graphy"+ :api #{metabase-enterprise.memoize-monitor.init}+ :uses #{analytics+ analytics-interface+ audit-app+ collections+ driver+ models+ util+ warehouse-schema+ warehouses+ enterprise/serialization}}+enterprise/metabot{:team "Metabot":api #{metabase-enterprise.metabot.api@@ -3458,6 +3473,7 @@metabase-enterprise.serialization.initmetabase-enterprise.serialization.schemametabase-enterprise.serialization.v2.backfill-ids}+ :friends #{enterprise/memoize-monitor}:uses #{analyticsapiapp-dbdiff --git a/enterprise/backend/src/metabase_enterprise/core/init.clj b/enterprise/backend/src/metabase_enterprise/core/init.cljindex 227749a7ad8..a93027616b4 100644--- a/enterprise/backend/src/metabase_enterprise/core/init.clj+++ b/enterprise/backend/src/metabase_enterprise/core/init.clj@@ -16,6 +16,7 @@[metabase-enterprise.database-replication.init][metabase-enterprise.dependencies.init][metabase-enterprise.gsheets.init]+ [metabase-enterprise.memoize-monitor.init][metabase-enterprise.metabot.init][metabase-enterprise.remote-sync.init][metabase-enterprise.scim.init]diff --git a/enterprise/backend/src/metabase_enterprise/memoize_monitor/init.clj b/enterprise/backend/src/metabase_enterprise/memoize_monitor/init.cljnew file mode 100644index 00000000000..a7845c925c2--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/memoize_monitor/init.clj@@ -0,0 +1,76 @@+(ns metabase-enterprise.memoize-monitor.init+ "Reports the size of a curated set of in-memory memoization caches as telemetry.++ Some memoization caches in the app are keyed on argument spaces that can grow large or unbounded (field IDs, table+ IDs, decrypted-secret ciphertexts, per-request collection-visibility configs, serialization paths, ...). Rather than+ instrument each cache at its definition site, this namespace references the vars that hold those caches directly and+ reads each cache's size on a periodic basis via a Prometheus `pull-collector`. For each monitored cache we emit:++ - `:metabase-memoize/cache-size` -- number of entries++ This lives in enterprise because the monitored caches span both OSS and enterprise modules+ ([[metabase-enterprise.serialization.dump]]); an OSS namespace cannot depend on enterprise code. OSS-only builds get+ no memoize-cache telemetry.++ Only caches built with [[clojure.core.memoize]] (`memo`, `ttl`, and+ [[metabase.app-db.core/memoize-for-application-db]], which is built on `memo`) can be measured, because their+ backing cache is reachable through the memoized function's metadata."+ (:require+ [metabase-enterprise.serialization.dump :as serialization.dump]+ [metabase.analytics-interface.core :as analytics.interface]+ [metabase.analytics.core :as analytics]+ [metabase.audit-app.impl :as audit-app.impl]+ [metabase.collections.models.collection :as collection]+ [metabase.driver.util :as driver.u]+ [metabase.models.interface :as mi]+ [metabase.util.log :as log]+ [metabase.warehouse-schema.models.field :as schema.field]+ [metabase.warehouses.models.database :as database])+ (:import+ (clojure.core.memoize PluggableMemoization)))++(set! *warn-on-reflection* true)++(def ^:private monitored-cache-vars+ "The vars holding the memoization caches whose sizes we export. Each must hold a [[clojure.core.memoize]]-backed+ function (see the namespace docstring)."+ [#'schema.field/field-id->table-id+ #'database/table-id->database-id+ #'database/db-id->router-db-id+ #'audit-app.impl/memoized-select-audit-entity*+ #'driver.u/memoized-supports?*+ #'driver.u/memoized-features*+ #'mi/cached-encrypted-json-out+ #'collection/can-access-root-collection?+ #'collection/visible-collection-ids*+ #'serialization.dump/serialization-sorted-map])++(defn- cache-object+ "The backing cache of a [[clojure.core.memoize]]-memoized function `f`, or `nil` if it isn't reachable. The returned+ object is a `clojure.core.cache` cache that is both `count`able and measurable."+ [f]+ (when-let [pm (some-> f meta :clojure.core.memoize/cache deref)]+ (when (instance? PluggableMemoization pm)+ (.cache ^PluggableMemoization pm))))++(defn- memoization-cache-stats+ "Map of `{:cache name, :entries n}` given a var holding a memoized function with a reachable backing cache, or `nil`+ otherwise."+ [cache-var]+ (try+ (when-let [cache (cache-object @cache-var)]+ (let [entries (count cache)]+ {:cache (str (symbol cache-var))+ :entries entries}))+ (catch Exception e+ (log/warn e "Error measuring memoization cache size" {:cache (str (symbol cache-var))}))))++(defn- all-cache-stats+ []+ (keep memoization-cache-stats monitored-cache-vars))++(defmethod analytics/pull-collector ::memoize-cache-sizes [_]+ {:min-interval-s (* 10 60)+ :f (fn []+ (doseq [{:keys [cache entries]} (all-cache-stats)]+ (analytics.interface/set-gauge! :metabase-memoize/cache-size {:cache cache} entries)))})diff --git a/enterprise/backend/src/metabase_enterprise/serialization/dump.clj b/enterprise/backend/src/metabase_enterprise/serialization/dump.cljindex 02404dc1eae..c0012e8daf2 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/dump.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/dump.clj@@ -1,6 +1,7 @@(ns metabase-enterprise.serialization.dump"Serialize entities into a directory structure of YAMLs."(:require+ [clojure.core.memoize :as memoize][clojure.edn :as edn][clojure.java.io :as io][metabase.util.yaml :as yaml]))@@ -23,7 +24,7 @@(compare (getter k1) (getter k2)))))(sorted-map)))-(def ^:private serialization-sorted-map (memoize serialization-sorted-map*))+(def ^:private serialization-sorted-map (memoize/memo serialization-sorted-map*))(defn serialization-deep-sort"Provide a deterministic sort for maps before serialization."diff --git a/enterprise/backend/test/metabase_enterprise/memoize_monitor/init_test.clj b/enterprise/backend/test/metabase_enterprise/memoize_monitor/init_test.cljnew file mode 100644index 00000000000..2f4255f7a7b--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/memoize_monitor/init_test.clj@@ -0,0 +1,29 @@+(ns metabase-enterprise.memoize-monitor.init-test+ (:require+ [clojure.core.memoize :as memoize]+ [clojure.test :refer :all]+ [metabase-enterprise.memoize-monitor.init :as memoize-monitor]))++(set! *warn-on-reflection* true)++(deftest cache-object-test+ (testing "returns the backing cache of a clojure.core.memoize function (countable + measurable)"+ (let [f (memoize/memo (fn [x] (* x x)))]+ (f 1) (f 2) (f 2)+ (is (= 2 (count (#'memoize-monitor/cache-object f))))))+ (testing "returns nil for a function whose backing cache isn't reachable"+ (is (nil? (#'memoize-monitor/cache-object (fn [x] x))))+ (is (nil? (#'memoize-monitor/cache-object (clojure.core/memoize (fn [x] x)))))))++(deftest cache-stats-entries-test+ (let [stats (#'memoize-monitor/all-cache-stats)+ by-cache (into {} (map (juxt :cache identity)) stats)]+ (testing "one stat per monitored cache, keyed by the var's fully-qualified symbol"+ (is (= (count @#'memoize-monitor/monitored-cache-vars)+ (count stats)))+ (is (contains? by-cache "metabase.warehouse-schema.models.field/field-id->table-id"))+ (is (contains? by-cache "metabase-enterprise.serialization.dump/serialization-sorted-map")))+ (testing "entry counts are always reported"+ (doseq [{:keys [cache entries]} stats]+ (is (string? cache))+ (is (and (integer? entries) (not (neg? entries))) (str cache " entries"))))))diff --git a/src/metabase/analytics/prometheus.clj b/src/metabase/analytics/prometheus.cljindex 0f1832543b8..aa2784f4ead 100644--- a/src/metabase/analytics/prometheus.clj+++ b/src/metabase/analytics/prometheus.clj@@ -773,7 +773,10 @@;; metaplow analytics metrics(prometheus/counter :metabase-metaplow/errors{:description "Metaplow event pipeline errors by stage."- :labels [:stage]})])+ :labels [:stage]})+ (prometheus/gauge :metabase-memoize/cache-size+ {:description "Number of entries currently held in a monitored in-memory memoization cache."+ :labels [:cache]})])(defn- quartz-collectors[]diff --git a/src/metabase/app_db/core.clj b/src/metabase/app_db/core.cljindex f55ab7dca89..cf38535a856 100644--- a/src/metabase/app_db/core.clj+++ b/src/metabase/app_db/core.clj@@ -9,6 +9,7 @@Namespaces outside of src/metabase/app_db/ should not use any metabase.app-db.* namespace but use this api namespace."(:refer-clojure :exclude [compile])(:require+ [clojure.core.memoize :as memoize][metabase.app-db.connection :as mdb.connection][metabase.app-db.connection-pool-setup :as mdb.connection-pool-setup][metabase.app-db.data-source :as mdb.data-source]@@ -153,10 +154,10 @@`:clojure.core.memoize/args-fn` instead; see [[metabase.driver.util/database->driver*]] for an example of how to dothis."[f]- (let [f* (memoize (fn [_application-db-id & args]- (apply f args)))]- (fn [& args]- (apply f* (unique-identifier) args))))+ (memoize/memo+ (vary-meta+ (fn [& args] (apply f args))+ assoc ::memoize/args-fn (fn [args] (cons (unique-identifier) args)))))(defn increment-app-db-unique-indentifier!"Increment the [[unique-identifier]] for the Metabase application DB. This effectively flushes all caches using it asdiff --git a/test/metabase/app_db/core_test.clj b/test/metabase/app_db/core_test.cljnew file mode 100644index 00000000000..4c73d03f553--- /dev/null+++ b/test/metabase/app_db/core_test.clj@@ -0,0 +1,23 @@+(ns metabase.app-db.core-test+ (:require+ [clojure.test :refer :all]+ [metabase.app-db.connection :as mdb.connection]+ [metabase.app-db.core :as mdb]))++(deftest memoize-for-application-db-test+ (let [calls (atom 0)+ f (mdb/memoize-for-application-db+ (fn [x]+ (swap! calls inc)+ (inc x)))]+ (testing "memoizes: the underlying fn runs once per distinct argument"+ (is (= 2 (f 1)))+ (is (= 2 (f 1)))+ (is (= 3 (f 2)))+ (is (= 2 @calls)))+ (testing "cache is keyed by application DB, so a different app DB recomputes"+ (mdb/with-application-db (assoc mdb.connection/*application-db* :id Integer/MAX_VALUE)+ (is (= 2 (f 1))))+ (is (= 3 @calls)))+ (testing "the cache is introspectable for telemetry (carries clojure.core.memoize cache metadata)"+ (is (some? (:clojure.core.memoize/cache (meta f)))))))