Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

[pivot] Optimize BE+FE pivot generation (#82811)

Oleksandr Yakushev5bcd89d33fcb
diff --git a/src/metabase/pivot/core.cljc b/src/metabase/pivot/core.cljc
index d541f9aa9a4..de43ec7e740 100644
--- a/src/metabase/pivot/core.cljc
+++ b/src/metabase/pivot/core.cljc
@@ -62,22 +62,12 @@
(let [breakout-indexes (range num-breakouts)]
(into [] (filter #(zero? (bit-and (bit-shift-left 1 %) (ensure-is-int pivot-group))) breakout-indexes))))))
-(defn- remove-item-by-index
- "Remove an item with the given `index` from collection `v`."
- [coll index-to-remove]
- (reduce-kv (fn [acc i x]
- (if (<= i index-to-remove)
- acc
- (conj acc x)))
- (subvec coll 0 index-to-remove)
- coll))
-
(defn- process-grouped-rows
"Processes rows for a specific pivot group value (k).
Returns a tuple of [active-breakout-indexes, rows-with-pivot-column-removed]."
[pivot-group rows pivot-group-index num-breakouts]
(let [active-indexes (get-active-breakout-indexes pivot-group num-breakouts)
- processed-rows (mapv #(remove-item-by-index % pivot-group-index) rows)]
+ processed-rows (mapv #(perf/remove-by-index % pivot-group-index) rows)]
[active-indexes processed-rows]))
(defn split-pivot-data
@@ -190,40 +180,38 @@
(defn- select-indexes
"Given a row, returns a subset of its values according to the provided indexes."
[row indexes]
- (mapv #(nth row %) indexes))
+ (perf/mapv #(nth row %) indexes))
(defn- build-values-by-key
"Creates a mapping from row and column indexes to the values, as well as
metadata used for conditional formatting and drill-throughs."
[rows cols row-indexes col-indexes val-indexes]
- (let [col-and-row-indexes (into (vec col-indexes) row-indexes)]
- (reduce
- (fn [acc row]
- (let [value-key (perf/mapv ensure-consistent-type (select-indexes row col-and-row-indexes))
- values (select-indexes row val-indexes)
- data (into []
- (map-indexed
- (fn [index value]
- {:value value
- :colIdx index}))
- row)
- dimensions (into []
- (keep-indexed (fn [index value]
- (when (= (:source (nth cols index)) "breakout")
- {:value value
- :colIdx index})))
- row)
- col-names (->> (select-indexes cols val-indexes)
- (map :name)
- (into []))]
- (assoc acc
- value-key
- {:values values
- :valueColNames col-names
- :data data
- :dimensions dimensions})))
- {}
- rows)))
+ (let [col-and-row-indexes (into (vec col-indexes) row-indexes)
+ col-names (perf/mapv :name (select-indexes cols val-indexes))
+ breakout-col-indexes (vec (keep-indexed (fn [index col]
+ (when (= (:source col) "breakout")
+ index))
+ cols))]
+ (persistent!
+ (reduce
+ (fn [acc row]
+ (let [value-key (perf/mapv #(ensure-consistent-type (nth row %)) col-and-row-indexes)
+ values (select-indexes row val-indexes)
+ data (perf/mapv-indexed (fn [^long index value]
+ {:value value
+ :colIdx index})
+ row)
+ dimensions (perf/mapv (fn [index]
+ {:value (nth row index)
+ :colIdx index})
+ breakout-col-indexes)]
+ (assoc! acc value-key
+ {:values values
+ :valueColNames col-names
+ :data data
+ :dimensions dimensions})))
+ (transient {})
+ rows))))
(defn- sort-orders-from-settings
[col-settings indexes]
@@ -550,22 +538,17 @@
`sum-where`, `share`). Without them, clicking a pivot cell for `CountIf([X] = Y)` drills without the `X = Y` filter
(#79023)."
[values-by-key index-values value-formatters val-indexes color-getter]
- (let [{:keys [values valueColNames data dimensions]} (get values-by-key index-values)
- formatted-values (format-values values value-formatters)]
- (if-not data
- formatted-values
- (map-indexed
- (fn [index value]
- (assoc value
- :clicked {:data data
- :dimensions dimensions
- :colIdx (nth val-indexes index)
- :value (nth values index)}
- :backgroundColor (color-getter
- (nth values index)
- index
- (nth valueColNames index))))
- formatted-values))))
+ (let [{:keys [values valueColNames data dimensions]} (get values-by-key index-values)]
+ (if data
+ (map-indexed (fn [index value]
+ {:value ((nth value-formatters index) value)
+ :clicked {:data data
+ :dimensions dimensions
+ :colIdx (nth val-indexes index)
+ :value value}
+ :backgroundColor (color-getter value index (nth valueColNames index))})
+ values)
+ (format-values values value-formatters))))
(defn- is-subtotal?
"Determines if a cell is a subtotal based on its position."
diff --git a/src/metabase/util/performance.cljc b/src/metabase/util/performance.cljc
index 71c798aa4dc..c91fa15f380 100644
--- a/src/metabase/util/performance.cljc
+++ b/src/metabase/util/performance.cljc
@@ -158,7 +158,15 @@
(persistent [_]
(LazilyPersistentVector/createOwning arr)))
:cljs
- (deftype SmallTransientImpl [^:mutable arr, ^:mutable cnt, f]))
+ (deftype SmallTransientImpl [^:mutable arr, ^:mutable cnt, f]
+ ITransientCollection
+ (-conj! [this x]
+ (aset arr cnt x)
+ (set! cnt (inc cnt))
+ this)
+
+ (-persistent! [_]
+ (PersistentVector. nil cnt 5 (.-EMPTY-NODE PersistentVector) arr nil))))
(defn- small-transient [n f]
;; Storing `f` in the transient itself is a hack to reduce lambda generation.
@@ -228,13 +236,13 @@
:cljs
(core/mapv f coll1 coll2 coll3 coll4))))
-#?(:clj
- (defn mapv-indexed
- "Like `clojure.core/map-indexed`, but returns a vector and uses Java iterators under the hood and optimized small
+(defn mapv-indexed
+ "Like `clojure.core/map-indexed`, but returns a vector and uses Java iterators under the hood and optimized small
transient vectors. Requires `f` to be a primitive function of (long, Object) -> Object."
- [f coll]
- (if (nil? coll)
- []
+ [f coll]
+ (if (nil? coll)
+ []
+ #?(:clj
(let [n (count coll)
it1 (.iterator ^Iterable coll)]
(loop [i 0, res (if (<= n 32)
@@ -243,7 +251,28 @@
(if (.hasNext it1)
;; Use .invokePrim directly to prevent index from autoboxing.
(recur (inc i) (conj! res (.invokePrim ^clojure.lang.IFn$LOO f i (.next it1))))
- (persistent! res)))))))
+ (persistent! res))))
+ :cljs
+ (let [n (count coll)]
+ (persistent!
+ (reduce-kv #(conj! %1 (f %2 %3))
+ (if (<= n 32)
+ (small-transient n identity)
+ (transient []))
+ coll))))))
+
+(defn remove-by-index
+ "Remove an item with the given `index` from vector `v`, returning a vector."
+ [v index]
+ (let [n (count v)]
+ (persistent!
+ (reduce #(if (= %2 index)
+ %1
+ (conj! %1 (nth v %2)))
+ (if (<= n 32)
+ (small-transient (dec n) identity)
+ (transient []))
+ (range (count v))))))
(defn run!
"Drop-in replacement for `clojure.core/run!`.