The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
[pivot] Optimize BE+FE pivot generation (#82811)
Oleksandr Yakushev5bcd89d33fcb
diff --git a/src/metabase/pivot/core.cljc b/src/metabase/pivot/core.cljcindex d541f9aa9a4..de43ec7e740 100644--- a/src/metabase/pivot/core.cljc+++ b/src/metabase/pivot/core.cljc@@ -62,22 +62,12 @@(let [breakout-indexes (range num-breakouts)](into [] (filter #(zero? (bit-and (bit-shift-left 1 %) (ensure-is-int pivot-group))) breakout-indexes))))))-(defn- remove-item-by-index- "Remove an item with the given `index` from collection `v`."- [coll index-to-remove]- (reduce-kv (fn [acc i x]- (if (<= i index-to-remove)- acc- (conj acc x)))- (subvec coll 0 index-to-remove)- coll))-(defn- process-grouped-rows"Processes rows for a specific pivot group value (k).Returns a tuple of [active-breakout-indexes, rows-with-pivot-column-removed]."[pivot-group rows pivot-group-index num-breakouts](let [active-indexes (get-active-breakout-indexes pivot-group num-breakouts)- processed-rows (mapv #(remove-item-by-index % pivot-group-index) rows)]+ processed-rows (mapv #(perf/remove-by-index % pivot-group-index) rows)][active-indexes processed-rows]))(defn split-pivot-data@@ -190,40 +180,38 @@(defn- select-indexes"Given a row, returns a subset of its values according to the provided indexes."[row indexes]- (mapv #(nth row %) indexes))+ (perf/mapv #(nth row %) indexes))(defn- build-values-by-key"Creates a mapping from row and column indexes to the values, as well asmetadata used for conditional formatting and drill-throughs."[rows cols row-indexes col-indexes val-indexes]- (let [col-and-row-indexes (into (vec col-indexes) row-indexes)]- (reduce- (fn [acc row]- (let [value-key (perf/mapv ensure-consistent-type (select-indexes row col-and-row-indexes))- values (select-indexes row val-indexes)- data (into []- (map-indexed- (fn [index value]- {:value value- :colIdx index}))- row)- dimensions (into []- (keep-indexed (fn [index value]- (when (= (:source (nth cols index)) "breakout")- {:value value- :colIdx index})))- row)- col-names (->> (select-indexes cols val-indexes)- (map :name)- (into []))]- (assoc acc- value-key- {:values values- :valueColNames col-names- :data data- :dimensions dimensions})))- {}- rows)))+ (let [col-and-row-indexes (into (vec col-indexes) row-indexes)+ col-names (perf/mapv :name (select-indexes cols val-indexes))+ breakout-col-indexes (vec (keep-indexed (fn [index col]+ (when (= (:source col) "breakout")+ index))+ cols))]+ (persistent!+ (reduce+ (fn [acc row]+ (let [value-key (perf/mapv #(ensure-consistent-type (nth row %)) col-and-row-indexes)+ values (select-indexes row val-indexes)+ data (perf/mapv-indexed (fn [^long index value]+ {:value value+ :colIdx index})+ row)+ dimensions (perf/mapv (fn [index]+ {:value (nth row index)+ :colIdx index})+ breakout-col-indexes)]+ (assoc! acc value-key+ {:values values+ :valueColNames col-names+ :data data+ :dimensions dimensions})))+ (transient {})+ rows))))(defn- sort-orders-from-settings[col-settings indexes]@@ -550,22 +538,17 @@`sum-where`, `share`). Without them, clicking a pivot cell for `CountIf([X] = Y)` drills without the `X = Y` filter(#79023)."[values-by-key index-values value-formatters val-indexes color-getter]- (let [{:keys [values valueColNames data dimensions]} (get values-by-key index-values)- formatted-values (format-values values value-formatters)]- (if-not data- formatted-values- (map-indexed- (fn [index value]- (assoc value- :clicked {:data data- :dimensions dimensions- :colIdx (nth val-indexes index)- :value (nth values index)}- :backgroundColor (color-getter- (nth values index)- index- (nth valueColNames index))))- formatted-values))))+ (let [{:keys [values valueColNames data dimensions]} (get values-by-key index-values)]+ (if data+ (map-indexed (fn [index value]+ {:value ((nth value-formatters index) value)+ :clicked {:data data+ :dimensions dimensions+ :colIdx (nth val-indexes index)+ :value value}+ :backgroundColor (color-getter value index (nth valueColNames index))})+ values)+ (format-values values value-formatters))))(defn- is-subtotal?"Determines if a cell is a subtotal based on its position."diff --git a/src/metabase/util/performance.cljc b/src/metabase/util/performance.cljcindex 71c798aa4dc..c91fa15f380 100644--- a/src/metabase/util/performance.cljc+++ b/src/metabase/util/performance.cljc@@ -158,7 +158,15 @@(persistent [_](LazilyPersistentVector/createOwning arr))):cljs- (deftype SmallTransientImpl [^:mutable arr, ^:mutable cnt, f]))+ (deftype SmallTransientImpl [^:mutable arr, ^:mutable cnt, f]+ ITransientCollection+ (-conj! [this x]+ (aset arr cnt x)+ (set! cnt (inc cnt))+ this)++ (-persistent! [_]+ (PersistentVector. nil cnt 5 (.-EMPTY-NODE PersistentVector) arr nil))))(defn- small-transient [n f];; Storing `f` in the transient itself is a hack to reduce lambda generation.@@ -228,13 +236,13 @@:cljs(core/mapv f coll1 coll2 coll3 coll4))))-#?(:clj- (defn mapv-indexed- "Like `clojure.core/map-indexed`, but returns a vector and uses Java iterators under the hood and optimized small+(defn mapv-indexed+ "Like `clojure.core/map-indexed`, but returns a vector and uses Java iterators under the hood and optimized smalltransient vectors. Requires `f` to be a primitive function of (long, Object) -> Object."- [f coll]- (if (nil? coll)- []+ [f coll]+ (if (nil? coll)+ []+ #?(:clj(let [n (count coll)it1 (.iterator ^Iterable coll)](loop [i 0, res (if (<= n 32)@@ -243,7 +251,28 @@(if (.hasNext it1);; Use .invokePrim directly to prevent index from autoboxing.(recur (inc i) (conj! res (.invokePrim ^clojure.lang.IFn$LOO f i (.next it1))))- (persistent! res)))))))+ (persistent! res))))+ :cljs+ (let [n (count coll)]+ (persistent!+ (reduce-kv #(conj! %1 (f %2 %3))+ (if (<= n 32)+ (small-transient n identity)+ (transient []))+ coll))))))++(defn remove-by-index+ "Remove an item with the given `index` from vector `v`, returning a vector."+ [v index]+ (let [n (count v)]+ (persistent!+ (reduce #(if (= %2 index)+ %1+ (conj! %1 (nth v %2)))+ (if (<= n 32)+ (small-transient (dec n) identity)+ (transient []))+ (range (count v))))))(defn run!"Drop-in replacement for `clojure.core/run!`.