Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

fix perf/mapv to handle infinite and large lazy sequences (#66984)

bryan29b1e70d2197
diff --git a/src/metabase/util/performance.cljc b/src/metabase/util/performance.cljc
index 4bebdb3ff99..b461273cffa 100644
--- a/src/metabase/util/performance.cljc
+++ b/src/metabase/util/performance.cljc
@@ -193,19 +193,17 @@
#?(:clj
(defn- smallest-count
- (^long [c1 c2] (min (count c1) (count c2)))
- (^long [c1 c2 c3] (min (count c1) (count c2) (count c3)))
- (^long [c1 c2 c3 c4] (min (count c1) (count c2) (count c3) (count c4)))))
+ (^long [c1 c2] (min (bounded-count 33 c1) (bounded-count 33 c2)))
+ (^long [c1 c2 c3] (min (bounded-count 33 c1) (bounded-count 33 c2) (bounded-count 33 c3)))
+ (^long [c1 c2 c3 c4] (min (bounded-count 33 c1) (bounded-count 33 c2) (bounded-count 33 c3) (bounded-count 33 c4)))))
(defn mapv
- "**Nearly** drop-in replacement for `clojure.core/mapv`.
+ "Drop-in replacement for `clojure.core/mapv`.
Iterates multiple collections more efficiently and uses Java iterators under the hood (the CLJ version). CLJS
- version is only optimized for a single collection arity.
-
- Checks the `count` of all inputs - DO NOT give this infinite sequences!"
+ version is only optimized for a single collection arity."
([f coll1]
- (let [n (count coll1)]
+ (let [n (bounded-count 33 coll1)]
(cond (= n 0) []
(<= n 32) (small-persistent! (reduce apply-and-small-conj! (small-transient n f) coll1))
:else (persistent! (reduce #(conj! %1 (f %2)) (transient []) coll1)))))
diff --git a/test/metabase/util/performance_test.cljc b/test/metabase/util/performance_test.cljc
index f438f2e58fe..5500c32188c 100644
--- a/test/metabase/util/performance_test.cljc
+++ b/test/metabase/util/performance_test.cljc
@@ -1,5 +1,8 @@
(ns metabase.util.performance-test
(:require
+ #?@(:clj [[clojure.test.check.clojure-test :refer [defspec]]
+ [clojure.test.check.properties :as prop]
+ [malli.generator :as mg]])
[clojure.test :refer [are deftest is testing]]
[metabase.util.performance :as perf]))
@@ -213,3 +216,113 @@
(testing "partial path exists"
(is (nil? (perf/get-in {:a 1} [:a :b])))
(is (= :fallback (perf/get-in {:a "not-a-map"} [:a :b] :fallback)))))
+
+#?(:clj
+ (defn- trap-seq
+ "Returns a lazy seq of `safe-count` elements (0 to safe-count - 1) followed by elements that throw on realization.
+ Used to verify that mapv doesn't over-realize lazy sequences."
+ [safe-count]
+ (concat (range safe-count)
+ (for [_ (range 10)]
+ (throw (ex-info "should not be realized" {:safe-count safe-count}))))))
+
+(deftest ^:parallel mapv-does-not-realize-lazy-seqs-test
+ (testing "Does not realize long or infinite sequences beyond what's needed (bug repro)"
+ (is (= [1 2]
+ (perf/mapv (comp first vector)
+ [1 2]
+ (concat (range 1000)
+ (for [_ (range 10)]
+ (throw (ex-info "you should not be here" {})))))))))
+
+#?(:clj
+ (defspec mapv-single-coll-equivalence 100
+ (prop/for-all [coll (mg/generator [:sequential :int])]
+ (= (mapv str coll)
+ (perf/mapv str coll)))))
+
+#?(:clj
+ (defspec mapv-two-coll-equivalence 100
+ (prop/for-all [c1 (mg/generator [:sequential :int])
+ c2 (mg/generator [:sequential :int])]
+ (= (mapv str c1 c2)
+ (perf/mapv str c1 c2)))))
+
+#?(:clj
+ (defspec mapv-three-coll-equivalence 100
+ (prop/for-all [c1 (mg/generator [:sequential :int])
+ c2 (mg/generator [:sequential :int])
+ c3 (mg/generator [:sequential :int])]
+ (= (mapv str c1 c2 c3)
+ (perf/mapv str c1 c2 c3)))))
+
+#?(:clj
+ (defspec mapv-four-coll-equivalence 100
+ (prop/for-all [c1 (mg/generator [:sequential :int])
+ c2 (mg/generator [:sequential :int])
+ c3 (mg/generator [:sequential :int])
+ c4 (mg/generator [:sequential :int])]
+ (= (mapv str c1 c2 c3 c4)
+ (perf/mapv str c1 c2 c3 c4)))))
+
+#?(:clj
+ (defspec smallest-count-does-not-over-realize-2-colls 100
+ (prop/for-all [short-len (mg/generator [:int {:min 0 :max 33}])
+ extra-len (mg/generator [:int {:min 34 :max 100}])]
+ (let [short-coll (vec (range short-len))
+ trap-coll (trap-seq (+ short-len extra-len))]
+ (= (#'perf/smallest-count short-coll trap-coll)
+ (min short-len extra-len))))))
+
+#?(:clj
+ (defspec mapv-does-not-over-realize-2-colls 1000
+ (prop/for-all [short-len (mg/generator [:int {:min 0 :max 50}])
+ extra-len (mg/generator [:int {:min 34 :max 100}])]
+ (let [short-coll (vec (range short-len))
+ trap-coll (trap-seq (+ short-len extra-len))]
+ (= (mapv vector short-coll trap-coll)
+ (perf/mapv vector short-coll trap-coll))))))
+
+#?(:clj
+ (defspec mapv-does-not-over-realize-3-colls 1000
+ (prop/for-all [short-len (mg/generator [:int {:min 0 :max 33}])
+ extra-len (mg/generator [:int {:min 34 :max 100}])]
+ (let [short-coll (vec (range short-len))
+ trap-coll (trap-seq (+ short-len extra-len))]
+ (= (mapv vector short-coll trap-coll trap-coll)
+ (perf/mapv vector short-coll trap-coll trap-coll))))))
+
+#?(:clj
+ (defspec mapv-does-not-over-realize-4-colls 1000
+ (prop/for-all [short-len (mg/generator [:int {:min 0 :max 33}])
+ extra-len (mg/generator [:int {:min 34 :max 100}])]
+ (let [short-coll (vec (range short-len))
+ trap-coll (trap-seq (+ short-len extra-len))]
+ (= (mapv vector short-coll trap-coll trap-coll trap-coll)
+ (perf/mapv vector short-coll trap-coll trap-coll trap-coll))))))
+
+#?(:clj
+ (defspec mapv-handles-infinite-seqs-2-colls 1000
+ (prop/for-all [coll (mg/generator [:vector {:min 0 :max 50} :int])]
+ (= (mapv vector coll (range))
+ (perf/mapv vector coll (range))))))
+
+#?(:clj
+ (defspec mapv-handles-infinite-seqs-3-colls 1000
+ (prop/for-all [coll (mg/generator [:vector {:min 0 :max 50} :int])]
+ (= (mapv vector coll (range) (iterate inc 0))
+ (perf/mapv vector coll (range) (iterate inc 0))))))
+
+#?(:clj
+ (defspec mapv-handles-infinite-seqs-4-colls 1000
+ (prop/for-all [coll (mg/generator [:vector {:min 0 :max 50} :int])]
+ (= (mapv vector coll (range) (iterate inc 0) (repeat 1))
+ (perf/mapv vector coll (range) (iterate inc 0) (repeat 1))))))
+
+#?(:clj
+ (defspec mapv-boundary-32-equivalence 1000
+ (prop/for-all [n (mg/generator [:int {:min 28 :max 36}])]
+ (let [c1 (vec (range n))
+ c2 (vec (range n))]
+ (= (mapv + c1 c2)
+ (perf/mapv + c1 c2))))))