The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Custom ref schema that reuses the cache (#68379)
Dan Stone9f5b23ecdd12
diff --git a/src/metabase/util/malli/registry.cljc b/src/metabase/util/malli/registry.cljcindex 5cff3ea936f..eea10eef694 100644--- a/src/metabase/util/malli/registry.cljc+++ b/src/metabase/util/malli/registry.cljc@@ -66,6 +66,11 @@(swap! cache assoc-in [k schema-key] (cache-entry v))v))))+(defn- cached-schema [schema]+ (if (mc/-reference? schema)+ (cached :ref schema #(mc/-into-schema (mc/-ref-schema) {} [schema] {}))+ (cached :schema schema #(mc/schema schema))))+(defn validator"Fetch a cached [[mc/validator]] for `schema`, creating one if needed. The cache is flushed whenever the registrychanges."@@ -73,7 +78,7 @@(letfn [(make-validator* [](try#_{:clj-kondo/ignore [:discouraged-var]}- (mc/validator schema)+ (mc/validator (cached-schema schema))(catch #?(:clj Throwable :cljs :default) e(throw (ex-info (str "Error making validator for " (pr-str schema) ":" (ex-message e)){:schema schema}@@ -102,7 +107,7 @@(try#_{:clj-kondo/ignore [:discouraged-var]}(let [validator* (validator schema)- explainer* (mc/explainer schema)]+ explainer* (mc/explainer (cached-schema schema))];; for valid values, it's significantly faster to just call the validator. Let's optimize for the 99.9%;; of calls whose values are valid.(fn schema-explainer [value]@@ -119,8 +124,54 @@[schema value]((explainer schema) value))+(defn- cached-ref-schema+ "Malli (as of 0.2.0) will re-allocate a specialised Schema object for a reference site+ as well as the referent's Schema each time it is referenced.++ The unique Schema object for the referent will then go on and re-allocate again for nested references,+ In malli 0.2.0 this can go on forever at runtime, resembling a memory leak.+ Even without significant runtime value complexity this very quickly leads to large heap memory usage+ as we retain validator roots in our cache.++ This custom :ref implementation dramatically reduces memory usage and OOM likely-hood:++ We allow references to reuse a cached Schema object if the following conditions hold:++ a. The registry is identical to the default, we refer to the same Schema object each time+ b. The reference itself has no attributes that specialise it (such as documentation).++ This means that most refs in our schemas will incur a pointer-like incremental cost,+ rather than an inlining-the-whole-subgraph type exponential cost.++ NOTE: At time of writing work is ongoing in malli to introduce knot-ties to validators/transformers.+ However, it appears we still see exponential memory scaling without this [[cached-ref-schema]] fix.+ We should revisit the need for this override with newer versions of malli."+ ([]+ (cached-ref-schema nil))+ ([{:keys [type-properties]}]+ (reify+ mc/AST+ (-from-ast [parent ast options] (mc/-from-value-ast parent ast options))+ mc/IntoSchema+ (-type [_] :ref)+ (-type-properties [_] type-properties)+ (-into-schema [_ properties [ref :as children] options]+ (mc/-check-children! :ref properties children 1 1)+ (when-not (mc/-reference? ref)+ (mc/-fail! ::mc/invalid-ref {:ref ref}))+ (let [reg (mc/-registry options)]+ ;; we only want to use our cache if this reference is equivalent to any other i,e:+ ;; - it must be placed in a context where the default registry is used.+ ;; (otherwise it might share a name with the globally registered schema but reference a different schema)+ ;; - it cannot have any properties that specialise the reference.+ (if (and (identical? reg mc/default-registry) (empty? properties))+ (cached :ref ref #(mc/-into-schema (mc/-ref-schema) {} children options))+ (mc/-into-schema (mc/-ref-schema) properties children options))))+ #?@(:cljs [IPrintWithWriter (-pr-writer [this writer opts] (mc/pr-writer-into-schema this writer opts))]))))+(defonce ^:private registry*(atom (merge (mc/default-schemas)+ {:ref (cached-ref-schema)}(mut/schemas)#?(:clj (malli.time/schemas)))))diff --git a/test/metabase/util/malli/registry_test.cljc b/test/metabase/util/malli/registry_test.cljcindex 3248224325d..5dca38a0be8 100644--- a/test/metabase/util/malli/registry_test.cljc+++ b/test/metabase/util/malli/registry_test.cljc@@ -12,7 +12,8 @@(testing (str "For things that aren't ever equal when you re-evaluate them (like Regex literals) maybe sure we do"" something smart to avoid creating infinite cache entries")(let [unique-schema [:re {:id (rand)} #"\d{4}"]]- (is (= 1 (with-returning-cache-miss-count+ ;; 2 misses as we cache both the Schema and validator.+ (is (= 2 (with-returning-cache-miss-count(mr/validate unique-schema "1234"))))(is (= 0 (with-returning-cache-miss-count(mr/validate unique-schema "1234")@@ -23,7 +24,8 @@(testing (str "For things that aren't ever equal when you re-evaluate them (like Regex literals) maybe sure we do"" something smart to avoid creating infinite cache entries")(let [unique-id (rand)]- (is (= 1 (with-returning-cache-miss-count+ ;; 2 misses as we cache both the Schema and validator.+ (is (= 2 (with-returning-cache-miss-count(mr/validate [:and {:id unique-id} :string [:re #"\d{4}"]] "1234"))))(let [validator-key-set (set (keys (:validator @@#'mr/cache)))](is (contains? validator-key-set@@ -35,7 +37,7 @@(mr/validate [:and {:id unique-id} :string [:re #"\d{4}"]] "1234")(mr/validate [:and {:id unique-id} :string [:re #"\d{4}"]] "1234")))"Calling validate with a previously cached schema does not miss cache")- (is (= 3+ (is (= 6(with-returning-cache-miss-count;; one(mr/validate [:and {:id unique-id} [:re #"\d{1}"] :string] "1234")