Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

[schema] Validation performance improvements (#64815)

Oleksandr Yakushev6c81e719585c
diff --git a/src/metabase/lib/schema/common.cljc b/src/metabase/lib/schema/common.cljc
index f4484b9b451..4ad8575c41d 100644
--- a/src/metabase/lib/schema/common.cljc
+++ b/src/metabase/lib/schema/common.cljc
@@ -8,7 +8,7 @@
[metabase.util.malli :as mu]
[metabase.util.malli.registry :as mr]
[metabase.util.memoize :as u.memo]
- [metabase.util.performance :refer [update-keys every? #?@(:clj [some])]]))
+ [metabase.util.performance :refer [update-keys every-key? #?@(:clj [some])]]))
(comment metabase.types.core/keep-me)
@@ -313,7 +313,7 @@
(defn- kebab-cased-map? [m]
(and (map? m)
- (every? kebab-cased-key? (keys m))))
+ (every-key? kebab-cased-key? m)))
(mr/def ::kebab-cased-map
[:fn
diff --git a/src/metabase/lib/schema/util.cljc b/src/metabase/lib/schema/util.cljc
index a6e6e8abe27..bc376806f67 100644
--- a/src/metabase/lib/schema/util.cljc
+++ b/src/metabase/lib/schema/util.cljc
@@ -1,11 +1,11 @@
(ns metabase.lib.schema.util
- (:refer-clojure :exclude [ref run! every? mapv empty? first second])
+ (:refer-clojure :exclude [ref run! every? mapv reduce empty? first second])
(:require
[medley.core :as m]
[metabase.lib.options :as lib.options]
[metabase.util.malli :as mu]
[metabase.util.malli.registry :as mr]
- [metabase.util.performance :as perf :refer [run! every? mapv empty? first second]]))
+ [metabase.util.performance :as perf :refer [run! every? mapv reduce empty? first second]]))
(declare collect-uuids*)
@@ -66,8 +66,7 @@
(defn- mbql-clauses?
[xs]
(and (sequential? xs)
- (or (empty? xs)
- (every? mbql-clause? xs))))
+ (every? mbql-clause? xs)))
(defn- opts-distinct-key [opts]
;; Using reduce-kv to remove namespaced keys and some other keys to perform the comparison. This is allegedly faster.
@@ -101,9 +100,21 @@
"Is a sequence of `mbql-clauses` distinct for the purposes of appearing in things like `:fields`, `:breakouts`, or
`:order-by`? (Are they distinct ignoring keys that aren't important such as namespaced keys and type info?)"
[mbql-clauses]
- (and (mbql-clauses? mbql-clauses)
- (or (< (count mbql-clauses) 2)
- (apply distinct? (map mbql-clause-distinct-key mbql-clauses)))))
+ (let [n (count mbql-clauses)]
+ (and (mbql-clauses? mbql-clauses)
+ (cond (< n 2) true
+ (= n 2) (not= (mbql-clause-distinct-key (first mbql-clauses))
+ (mbql-clause-distinct-key (second mbql-clauses)))
+ ;; This clause reimplements a specialized version of "all unique by key?" that shortcurcuits on
+ ;; duplicate element.
+ :else (let [found-dup? (true? (reduce (fn [seen x]
+ (let [x-key (mbql-clause-distinct-key x)]
+ (if (contains? seen x-key)
+ (reduced true)
+ (conj! seen x-key))))
+ (transient #{})
+ mbql-clauses))]
+ (not found-dup?))))))
(mr/def ::distinct-mbql-clauses
[:fn
diff --git a/src/metabase/util/performance.cljc b/src/metabase/util/performance.cljc
index 3e906a5a1d7..b67bf5d06f1 100644
--- a/src/metabase/util/performance.cljc
+++ b/src/metabase/util/performance.cljc
@@ -274,6 +274,11 @@
[f coll]
(unreduced (reduce #(if (f %2) true (reduced false)) true coll)))
+(defn every-key?
+ "Efficiently check if every key in map `m` matched predicate `f`."
+ [f m]
+ (reduce-kv (fn [_ k _] (if (f k) true (reduced false))) true m))
+
(defn concat
"Like `clojure.core/concat` but accumulates the result into a vector. NOT a drop-in replacement."
([a b]