Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

[perf] Implement `dropv` and use it in metabase.util.match (#82010)

Oleksandr Yakushev7c6dad4ec9f0
diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.edn
index 6e84284720f..810a47aaa7f 100644
--- a/.clj-kondo/config.edn
+++ b/.clj-kondo/config.edn
@@ -281,10 +281,6 @@
instaparse.core/transform
(metabase.server.streaming-response/streaming-response)
(metabase.driver.druid.query-processor-test/druid-query-returning-rows)
- (metabase.util.match/match-one)
- (metabase.util.match/match-many)
- (metabase.util.match/replace-in)
- (metabase.util.match/replace)
(metabase.driver-api.core/match-one)
(metabase.driver-api.core/match-many)
(metabase.driver-api.core/replace)
diff --git a/src/metabase/analytics/impl.cljs b/src/metabase/analytics/impl.cljs
index 2e48a992b20..c2664f577ea 100644
--- a/src/metabase/analytics/impl.cljs
+++ b/src/metabase/analytics/impl.cljs
@@ -8,7 +8,8 @@
along on the next flush as an [[dropped-metric]] :inc event so dashboards can tell 'quiet
system' apart from 'we dropped half your events'."
(:require
- [metabase.analytics-interface.core :as analytics.interface]))
+ [metabase.analytics-interface.core :as analytics.interface]
+ [metabase.util.performance :refer [dropv]]))
(def ^:private flush-interval-ms 5000)
@@ -23,7 +24,7 @@
[{:keys [events dropped]} event capacity]
(let [events' (conj events event)
overflow (max 0 (- (count events') capacity))]
- {:events (if (pos? overflow) (vec (drop overflow events')) events')
+ {:events (if (pos? overflow) (dropv overflow events') events')
:dropped (+ dropped overflow)}))
(defn- take-pending
diff --git a/src/metabase/channel/render/body.clj b/src/metabase/channel/render/body.clj
index 4baf43318c0..d8930ecb06b 100644
--- a/src/metabase/channel/render/body.clj
+++ b/src/metabase/channel/render/body.clj
@@ -908,7 +908,7 @@
"Swap the first two columns in data, reordering both :cols and :rows."
[data]
(-> data
- (update :cols (fn [cs] (vec (cons (second cs) (cons (first cs) (drop 2 cs))))))
+ (update :cols (fn [cs] (into [(second cs) (first cs)] (drop 2 cs))))
(update :rows (fn [rs] (mapv (fn [r] (let [v (vec r)] (into [(v 1) (v 0)] (subvec v 2)))) rs)))))
(defn- normalize-funnel-data
diff --git a/src/metabase/lib/util.cljc b/src/metabase/lib/util.cljc
index 237de9bbefe..5a68195023a 100644
--- a/src/metabase/lib/util.cljc
+++ b/src/metabase/lib/util.cljc
@@ -23,7 +23,7 @@
[metabase.util.malli :as mu]
[metabase.util.malli.registry :as mr]
[metabase.util.match :as match]
- [metabase.util.performance :refer [every? mapv select-keys update-keys some get-in #?(:clj for)]]))
+ [metabase.util.performance :refer [every? mapv select-keys update-keys some get-in dropv #?(:clj for)]]))
#?(:clj
(set! *warn-on-reflection* true))
@@ -221,7 +221,7 @@
(cond-> m
(contains? m legacy-key) (update legacy-key #(if (and (vector? %)
(= (first %) :and))
- (vec (drop 1 %))
+ (dropv 1 %)
[%]))
(contains? m legacy-key) (set/rename-keys {legacy-key mbql5-key})))
diff --git a/src/metabase/lib_metric/ast/build.cljc b/src/metabase/lib_metric/ast/build.cljc
index 2fc9ad8a670..cb9095ec0ff 100644
--- a/src/metabase/lib_metric/ast/build.cljc
+++ b/src/metabase/lib_metric/ast/build.cljc
@@ -431,7 +431,7 @@
:else
(let [op (arithmetic-operator expression)
- children (drop 2 expression)]
+ children (perf/dropv 2 expression)]
{:node/type :expression/arithmetic
:operator op
:children (perf/mapv #(build-expression-ast % metadata-provider filters projections) children)})))
diff --git a/src/metabase/lib_metric/filter.cljc b/src/metabase/lib_metric/filter.cljc
index c07e097b0d9..8087f7bda54 100644
--- a/src/metabase/lib_metric/filter.cljc
+++ b/src/metabase/lib_metric/filter.cljc
@@ -323,7 +323,7 @@
;; Standard operators with values in positions 3+
{:operator operator
:dimension dimension
- :values (vec (drop 3 filter-clause))}))))))))
+ :values (perf/dropv 3 filter-clause)}))))))))
;;; -------------------------------------------------- String Filters --------------------------------------------------
@@ -370,7 +370,7 @@
;; Equality operators
{:operator operator
:dimension dimension
- :values (vec (drop 3 filter-clause))
+ :values (perf/dropv 3 filter-clause)
:options {}}))))))))
;;; -------------------------------------------------- Coordinate Filters --------------------------------------------------
@@ -421,7 +421,7 @@
{:operator operator
:dimension dimension
:longitude-dimension nil
- :values (vec (drop 3 filter-clause))}))))))))
+ :values (perf/dropv 3 filter-clause)}))))))))
;;; -------------------------------------------------- Specific Date Filters --------------------------------------------------
@@ -581,7 +581,7 @@
{:operator operator
:dimension dimension
:unit unit
- :values (vec (drop 3 filter-clause))})))))))))
+ :values (perf/dropv 3 filter-clause)})))))))))
;;; -------------------------------------------------- Time Filters --------------------------------------------------
diff --git a/src/metabase/metabot/tools/resources.clj b/src/metabase/metabot/tools/resources.clj
index 5d9a825c531..02e35164d3c 100644
--- a/src/metabase/metabot/tools/resources.clj
+++ b/src/metabase/metabot/tools/resources.clj
@@ -115,7 +115,7 @@
(if (= pages 1) " page." " pages."))
{:page page :pages pages})))
start (* (dec page) page-size)]
- {:items (vec (take page-size (drop start items)))
+ {:items (subvec items start (min total (+ start page-size)))
:total total
:page page
:pages pages}))
diff --git a/src/metabase/util/match.clj b/src/metabase/util/match.clj
index 98e0bd492b2..9034e733f16 100644
--- a/src/metabase/util/match.clj
+++ b/src/metabase/util/match.clj
@@ -4,7 +4,7 @@
(:refer-clojure :exclude [every? run! some mapv replace empty?])
(:require
[metabase.util.match.impl]
- [metabase.util.performance :as perf :refer [empty? every? mapv run! some]]))
+ [metabase.util.performance :as perf :refer [empty? every? mapv run! some dropv]]))
(defn- parse-pattern
"Parse a pattern vector into bindings and conditions"
@@ -78,7 +78,7 @@
`metabase.util.match.impl/count=) s cnt)
{:depends-on s})))
(when rest-part
- (process-pattern rest-part `(into [] (drop ~cnt) ~s) bindings conditions false)))
+ (process-pattern rest-part `(dropv ~cnt ~s) bindings conditions false)))
:map (let [s (if (symbol? value) value (gensym "map"))]
(vswap! bindings conj [s `(metabase.util.match.impl/map! ~value)])
(run! (fn [[k v]]
diff --git a/src/metabase/util/performance.cljc b/src/metabase/util/performance.cljc
index 87378c45746..71c798aa4dc 100644
--- a/src/metabase/util/performance.cljc
+++ b/src/metabase/util/performance.cljc
@@ -349,6 +349,15 @@
(mapv (fn [_] (mapv #(.next ^Iterator %) its))
(first coll-of-colls)))))
+(defn dropv
+ "Like `clojure.core/drop`, but always returns a vector, and calls `clojure.core/subvec` when invoked on vectors (so,
+ it is very cheap, but shares the structure with original vector, see its docstring)."
+ ([n] (drop n))
+ ([n coll]
+ (cond (vector? coll) (if (> (count coll) n) (subvec coll n) [])
+ (nil? coll) []
+ :else (into [] (drop n) coll))))
+
(defn select-keys
"Drop-in replacement for `clojure.walk/select-keys`, but much more efficient."
[m keyseq]
diff --git a/test/metabase/comments/api_test.clj b/test/metabase/comments/api_test.clj
index 7ee41d78286..e1326135da5 100644
--- a/test/metabase/comments/api_test.clj
+++ b/test/metabase/comments/api_test.clj
@@ -8,6 +8,7 @@
[metabase.permissions.core :as perms]
[metabase.test :as mt]
[metabase.util :as u]
+ [metabase.util.performance :refer [dropv]]
[toucan2.core :as t2]))
(defn- relaxed-re [& s]
@@ -23,7 +24,7 @@
(name (first node)))
block? #{"paragraph"}
attrs (when (map? (second node)) (second node))
- content (if attrs (drop 2 node) (drop 1 node))]
+ content (dropv (if attrs 2 1) node)]
(u/remove-nils
{:type tag
:attrs (cond-> attrs
diff --git a/test/metabase/metrics/api_dimension_test.clj b/test/metabase/metrics/api_dimension_test.clj
index 6c1405b52ec..de315183ad4 100644
--- a/test/metabase/metrics/api_dimension_test.clj
+++ b/test/metabase/metrics/api_dimension_test.clj
@@ -9,6 +9,7 @@
[metabase.metrics.core :as metrics]
[metabase.test :as mt]
[metabase.test.fixtures :as fixtures]
+ [metabase.util.performance :refer [dropv]]
[toucan2.core :as t2]))
(use-fixtures :once (fixtures/initialize :db :web-server :test-users))
@@ -573,7 +574,7 @@
(with-seeded-metric [metric]
(let [ids (->> (mt/user-http-request :crowberto :get 200 (str "metric/" (:id metric) "/dimension"))
:added (mapv :id))
- [listed unlisted] [(vec (reverse (take 2 ids))) (vec (drop 2 ids))]
+ [listed unlisted] [(vec (reverse (take 2 ids))) (dropv 2 ids)]
resp (mt/user-http-request :crowberto :post 200
(str "metric/" (:id metric) "/dimension/reorder")
{:dimension_ids listed})]
diff --git a/test/metabase/util/performance_test.cljc b/test/metabase/util/performance_test.cljc
index ea3411dfa7d..f6160a4d2bb 100644
--- a/test/metabase/util/performance_test.cljc
+++ b/test/metabase/util/performance_test.cljc
@@ -61,6 +61,18 @@
(is (= [] (mapv-via-run! inc [])))
(is (= [1 2 3 4 5] (mapv-via-run! inc (range 5)))))
+(deftest dropv-test
+ (is (= [3 4 5] (perf/dropv 2 [1 2 3 4 5])))
+ (is (= [1 2 3] (perf/dropv 0 [1 2 3])))
+ (is (= [] (perf/dropv 3 [1 2 3])))
+ (is (= [] (perf/dropv 5 [1 2 3])))
+ (testing "nil input"
+ (is (= [] (perf/dropv 0 nil)))
+ (is (= [] (perf/dropv 5 nil))))
+ (testing "non-vectors"
+ (is (= [3 4 5] (perf/dropv 2 '(1 2 3 4 5))))
+ (is (vector? (perf/dropv 2 '(1 2 3 4 5))))))
+
#?(:clj
(deftest ^:parallel transpose-test
(is (= [[1 2 3 4] [1 2 3 4] [1 2 3 4] [1 2 3 4] [1 2 3 4]]