The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
fix perf/mapv to handle infinite and large lazy sequences (#66984)
bryan29b1e70d2197
diff --git a/src/metabase/util/performance.cljc b/src/metabase/util/performance.cljcindex 4bebdb3ff99..b461273cffa 100644--- a/src/metabase/util/performance.cljc+++ b/src/metabase/util/performance.cljc@@ -193,19 +193,17 @@#?(:clj(defn- smallest-count- (^long [c1 c2] (min (count c1) (count c2)))- (^long [c1 c2 c3] (min (count c1) (count c2) (count c3)))- (^long [c1 c2 c3 c4] (min (count c1) (count c2) (count c3) (count c4)))))+ (^long [c1 c2] (min (bounded-count 33 c1) (bounded-count 33 c2)))+ (^long [c1 c2 c3] (min (bounded-count 33 c1) (bounded-count 33 c2) (bounded-count 33 c3)))+ (^long [c1 c2 c3 c4] (min (bounded-count 33 c1) (bounded-count 33 c2) (bounded-count 33 c3) (bounded-count 33 c4)))))(defn mapv- "**Nearly** drop-in replacement for `clojure.core/mapv`.+ "Drop-in replacement for `clojure.core/mapv`.Iterates multiple collections more efficiently and uses Java iterators under the hood (the CLJ version). CLJS- version is only optimized for a single collection arity.-- Checks the `count` of all inputs - DO NOT give this infinite sequences!"+ version is only optimized for a single collection arity."([f coll1]- (let [n (count coll1)]+ (let [n (bounded-count 33 coll1)](cond (= n 0) [](<= n 32) (small-persistent! (reduce apply-and-small-conj! (small-transient n f) coll1)):else (persistent! (reduce #(conj! %1 (f %2)) (transient []) coll1)))))diff --git a/test/metabase/util/performance_test.cljc b/test/metabase/util/performance_test.cljcindex f438f2e58fe..5500c32188c 100644--- a/test/metabase/util/performance_test.cljc+++ b/test/metabase/util/performance_test.cljc@@ -1,5 +1,8 @@(ns metabase.util.performance-test(:require+ #?@(:clj [[clojure.test.check.clojure-test :refer [defspec]]+ [clojure.test.check.properties :as prop]+ [malli.generator :as mg]])[clojure.test :refer [are deftest is testing]][metabase.util.performance :as perf]))@@ -213,3 +216,113 @@(testing "partial path exists"(is (nil? (perf/get-in {:a 1} [:a :b])))(is (= :fallback (perf/get-in {:a "not-a-map"} [:a :b] :fallback)))))++#?(:clj+ (defn- trap-seq+ "Returns a lazy seq of `safe-count` elements (0 to safe-count - 1) followed by elements that throw on realization.+ Used to verify that mapv doesn't over-realize lazy sequences."+ [safe-count]+ (concat (range safe-count)+ (for [_ (range 10)]+ (throw (ex-info "should not be realized" {:safe-count safe-count}))))))++(deftest ^:parallel mapv-does-not-realize-lazy-seqs-test+ (testing "Does not realize long or infinite sequences beyond what's needed (bug repro)"+ (is (= [1 2]+ (perf/mapv (comp first vector)+ [1 2]+ (concat (range 1000)+ (for [_ (range 10)]+ (throw (ex-info "you should not be here" {})))))))))++#?(:clj+ (defspec mapv-single-coll-equivalence 100+ (prop/for-all [coll (mg/generator [:sequential :int])]+ (= (mapv str coll)+ (perf/mapv str coll)))))++#?(:clj+ (defspec mapv-two-coll-equivalence 100+ (prop/for-all [c1 (mg/generator [:sequential :int])+ c2 (mg/generator [:sequential :int])]+ (= (mapv str c1 c2)+ (perf/mapv str c1 c2)))))++#?(:clj+ (defspec mapv-three-coll-equivalence 100+ (prop/for-all [c1 (mg/generator [:sequential :int])+ c2 (mg/generator [:sequential :int])+ c3 (mg/generator [:sequential :int])]+ (= (mapv str c1 c2 c3)+ (perf/mapv str c1 c2 c3)))))++#?(:clj+ (defspec mapv-four-coll-equivalence 100+ (prop/for-all [c1 (mg/generator [:sequential :int])+ c2 (mg/generator [:sequential :int])+ c3 (mg/generator [:sequential :int])+ c4 (mg/generator [:sequential :int])]+ (= (mapv str c1 c2 c3 c4)+ (perf/mapv str c1 c2 c3 c4)))))++#?(:clj+ (defspec smallest-count-does-not-over-realize-2-colls 100+ (prop/for-all [short-len (mg/generator [:int {:min 0 :max 33}])+ extra-len (mg/generator [:int {:min 34 :max 100}])]+ (let [short-coll (vec (range short-len))+ trap-coll (trap-seq (+ short-len extra-len))]+ (= (#'perf/smallest-count short-coll trap-coll)+ (min short-len extra-len))))))++#?(:clj+ (defspec mapv-does-not-over-realize-2-colls 1000+ (prop/for-all [short-len (mg/generator [:int {:min 0 :max 50}])+ extra-len (mg/generator [:int {:min 34 :max 100}])]+ (let [short-coll (vec (range short-len))+ trap-coll (trap-seq (+ short-len extra-len))]+ (= (mapv vector short-coll trap-coll)+ (perf/mapv vector short-coll trap-coll))))))++#?(:clj+ (defspec mapv-does-not-over-realize-3-colls 1000+ (prop/for-all [short-len (mg/generator [:int {:min 0 :max 33}])+ extra-len (mg/generator [:int {:min 34 :max 100}])]+ (let [short-coll (vec (range short-len))+ trap-coll (trap-seq (+ short-len extra-len))]+ (= (mapv vector short-coll trap-coll trap-coll)+ (perf/mapv vector short-coll trap-coll trap-coll))))))++#?(:clj+ (defspec mapv-does-not-over-realize-4-colls 1000+ (prop/for-all [short-len (mg/generator [:int {:min 0 :max 33}])+ extra-len (mg/generator [:int {:min 34 :max 100}])]+ (let [short-coll (vec (range short-len))+ trap-coll (trap-seq (+ short-len extra-len))]+ (= (mapv vector short-coll trap-coll trap-coll trap-coll)+ (perf/mapv vector short-coll trap-coll trap-coll trap-coll))))))++#?(:clj+ (defspec mapv-handles-infinite-seqs-2-colls 1000+ (prop/for-all [coll (mg/generator [:vector {:min 0 :max 50} :int])]+ (= (mapv vector coll (range))+ (perf/mapv vector coll (range))))))++#?(:clj+ (defspec mapv-handles-infinite-seqs-3-colls 1000+ (prop/for-all [coll (mg/generator [:vector {:min 0 :max 50} :int])]+ (= (mapv vector coll (range) (iterate inc 0))+ (perf/mapv vector coll (range) (iterate inc 0))))))++#?(:clj+ (defspec mapv-handles-infinite-seqs-4-colls 1000+ (prop/for-all [coll (mg/generator [:vector {:min 0 :max 50} :int])]+ (= (mapv vector coll (range) (iterate inc 0) (repeat 1))+ (perf/mapv vector coll (range) (iterate inc 0) (repeat 1))))))++#?(:clj+ (defspec mapv-boundary-32-equivalence 1000+ (prop/for-all [n (mg/generator [:int {:min 28 :max 36}])]+ (let [c1 (vec (range n))+ c2 (vec (range n))]+ (= (mapv + c1 c2)+ (perf/mapv + c1 c2))))))