Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

[query] Multiple query prep optimizations (#84720)

Oleksandr Yakushev48cebd66e82e
diff --git a/src/metabase/lib/query.cljc b/src/metabase/lib/query.cljc
index 1833de7b144..062fe2cbf4b 100644
--- a/src/metabase/lib/query.cljc
+++ b/src/metabase/lib/query.cljc
@@ -123,8 +123,7 @@
[:field
(opts :guard (and (map? opts) (not (and (:base-type opts) (:effective-type opts)))))
(id :guard (and (integer? id) (pos? id)))]
- (when-not (some #{:mbql/stage-metadata} &parents)
- id))]
+ id)]
;; "pre-warm" the metadata provider
(do (lib.metadata/bulk-metadata metadata-provider :metadata/column field-ids)
(match/replace x
@@ -132,21 +131,20 @@
(options :guard (and (map? options) (not (and (:base-type options)
(:effective-type options)))))
(id :guard pos-int?)]
- (if (some #{:mbql/stage-metadata} &parents)
- &match
- (update &match 1 merge
- ;; TODO: For brush filters, query with different base type as in metadata is sent from FE. In that
- ;; case no change is performed. Find a way how to handle this properly!
- (when-not (and (some? (:base-type options))
- (not= (:base-type options)
- (:base-type (lib.metadata/field metadata-provider id))))
- ;; Following key is used to track which base-types we added during `query` call. It is used in
- ;; [[metabase.lib.convert/options->legacy-MBQL]] to remove those, so query after conversion
- ;; as legacy -> MBQL 5 -> legacy looks closer to the original.
- (merge (when-not (contains? options :base-type)
- {:lib/transformation-added-base-type true})
- (-> (lib.metadata/field metadata-provider id)
- (select-keys [:base-type :effective-type]))))))))
+ (let [{:keys [base-type]} options
+ mp-field (lib.metadata/field metadata-provider id)]
+ ;; TODO: For brush filters, query with different base type as in metadata is sent from FE. In that
+ ;; case no change is performed. Find a way how to handle this properly!
+ (if (and base-type (not= base-type (:base-type mp-field)))
+ &match
+ ;; Following key is used to track which base-types we added during `query` call. It is used in
+ ;; [[metabase.lib.convert/options->legacy-MBQL]] to remove those, so query after conversion
+ ;; as legacy -> MBQL 5 -> legacy looks closer to the original.
+ (let [new-opts (-> options
+ (cond-> (nil? base-type) (assoc :lib/transformation-added-base-type true))
+ (m/assoc-some :base-type (:base-type mp-field))
+ (m/assoc-some :effective-type (:effective-type mp-field)))]
+ [:field new-opts id])))))
x))
(mu/defn query-with-stages :- ::lib.schema/query
diff --git a/src/metabase/query_permissions/impl.clj b/src/metabase/query_permissions/impl.clj
index dacfdcb0749..14ce505289c 100644
--- a/src/metabase/query_permissions/impl.clj
+++ b/src/metabase/query_permissions/impl.clj
@@ -6,7 +6,6 @@
TODO -- does this belong HERE or in the `permissions` module?"
(:require
[clojure.set :as set]
- [clojure.walk :as walk]
[metabase.api.common :as api]
;; legacy usage -- do not use in new code
^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.normalize :as mbql.normalize]
@@ -32,6 +31,7 @@
[metabase.util.malli.registry :as mr]
[metabase.util.malli.schema :as ms]
[metabase.util.match :as match]
+ [metabase.util.performance :as perf]
[toucan2.core :as t2]))
(set! *warn-on-reflection* true)
@@ -240,7 +240,7 @@
"Return the union of all the `:query-permissions/referenced-card-ids` sets anywhere in the query."
[query]
(let [all-ids (atom #{})]
- (walk/postwalk
+ (perf/postwalk
(fn [form]
(when (map? form)
(when-let [ids (not-empty (:query-permissions/referenced-card-ids form))]
diff --git a/src/metabase/query_processor/reducible.clj b/src/metabase/query_processor/reducible.clj
index 020a401042f..86ed5514091 100644
--- a/src/metabase/query_processor/reducible.clj
+++ b/src/metabase/query_processor/reducible.clj
@@ -94,24 +94,22 @@
[primary-rf :- ::qp.schema/rf
additional-rfs :- [:sequential ::qp.schema/rf]
combine :- ifn?]
- (let [additional-accs (volatile! (perf/mapv (fn [rf] (rf))
- additional-rfs))]
+ (let [additional-rfs (vec additional-rfs)
+ ^objects additional-accs (perf/mapa (fn [rf] (rf)) additional-rfs)]
(fn combine-additional-reducing-fns-rf*
([] (primary-rf))
([acc]
- (let [additional-results (perf/mapv (fn [rf acc]
- (rf (unreduced acc)))
- additional-rfs
- @additional-accs)]
- (apply combine acc additional-results)))
+ (areduce additional-accs i _ nil
+ (let [rf (nth additional-rfs i)
+ acc (aget additional-accs i)]
+ (aset additional-accs i (rf (unreduced acc)))))
+ (apply combine acc (clojure.lang.LazilyPersistentVector/createOwning additional-accs)))
([acc x]
- (vswap! additional-accs (fn [accs]
- (perf/mapv (fn [rf acc]
- (if (reduced? acc)
- acc
- (rf acc x)))
- additional-rfs
- accs)))
+ (areduce additional-accs i _ nil
+ (let [rf (nth additional-rfs i)
+ acc (aget additional-accs i)]
+ (when-not (reduced? acc)
+ (aset additional-accs i (rf acc x)))))
(primary-rf acc x)))))
diff --git a/src/metabase/util/performance.cljc b/src/metabase/util/performance.cljc
index bab45076aa6..2b00550a9c9 100644
--- a/src/metabase/util/performance.cljc
+++ b/src/metabase/util/performance.cljc
@@ -356,19 +356,16 @@
(throw (ex-info "Position should be in the range [0; N] of vector." {})))
(let [v (vec v)
n (count v)]
- (if (= pos n)
- (conj v el)
- (let [i (volatile! -1)]
- (persistent! (reduce (fn [acc x]
- (vswap! i inc)
- (conj! (cond-> acc
- (= @i pos) (conj! el))
- x))
- #?(:clj (if (<= n 32)
- (small-transient (inc n) identity)
- (transient []))
- :cljs (transient []))
- v))))))
+ (loop [i 0, acc #?(:clj (if (<= n 32)
+ (small-transient (inc n) identity)
+ (transient []))
+ :cljs (transient []))]
+ (if (< i pos)
+ (recur (inc i) (conj! acc (nth v i)))
+ (loop [i pos, acc (conj! acc el)]
+ (if (< i n)
+ (recur (inc i) (conj! acc (nth v i)))
+ (persistent! acc)))))))
#?(:clj
(defn transpose