The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
[query] Multiple query prep optimizations (#84720)
Oleksandr Yakushev48cebd66e82e
diff --git a/src/metabase/lib/query.cljc b/src/metabase/lib/query.cljcindex 1833de7b144..062fe2cbf4b 100644--- a/src/metabase/lib/query.cljc+++ b/src/metabase/lib/query.cljc@@ -123,8 +123,7 @@[:field(opts :guard (and (map? opts) (not (and (:base-type opts) (:effective-type opts)))))(id :guard (and (integer? id) (pos? id)))]- (when-not (some #{:mbql/stage-metadata} &parents)- id))]+ id)];; "pre-warm" the metadata provider(do (lib.metadata/bulk-metadata metadata-provider :metadata/column field-ids)(match/replace x@@ -132,21 +131,20 @@(options :guard (and (map? options) (not (and (:base-type options)(:effective-type options)))))(id :guard pos-int?)]- (if (some #{:mbql/stage-metadata} &parents)- &match- (update &match 1 merge- ;; TODO: For brush filters, query with different base type as in metadata is sent from FE. In that- ;; case no change is performed. Find a way how to handle this properly!- (when-not (and (some? (:base-type options))- (not= (:base-type options)- (:base-type (lib.metadata/field metadata-provider id))))- ;; Following key is used to track which base-types we added during `query` call. It is used in- ;; [[metabase.lib.convert/options->legacy-MBQL]] to remove those, so query after conversion- ;; as legacy -> MBQL 5 -> legacy looks closer to the original.- (merge (when-not (contains? options :base-type)- {:lib/transformation-added-base-type true})- (-> (lib.metadata/field metadata-provider id)- (select-keys [:base-type :effective-type]))))))))+ (let [{:keys [base-type]} options+ mp-field (lib.metadata/field metadata-provider id)]+ ;; TODO: For brush filters, query with different base type as in metadata is sent from FE. In that+ ;; case no change is performed. Find a way how to handle this properly!+ (if (and base-type (not= base-type (:base-type mp-field)))+ &match+ ;; Following key is used to track which base-types we added during `query` call. It is used in+ ;; [[metabase.lib.convert/options->legacy-MBQL]] to remove those, so query after conversion+ ;; as legacy -> MBQL 5 -> legacy looks closer to the original.+ (let [new-opts (-> options+ (cond-> (nil? base-type) (assoc :lib/transformation-added-base-type true))+ (m/assoc-some :base-type (:base-type mp-field))+ (m/assoc-some :effective-type (:effective-type mp-field)))]+ [:field new-opts id])))))x))(mu/defn query-with-stages :- ::lib.schema/querydiff --git a/src/metabase/query_permissions/impl.clj b/src/metabase/query_permissions/impl.cljindex dacfdcb0749..14ce505289c 100644--- a/src/metabase/query_permissions/impl.clj+++ b/src/metabase/query_permissions/impl.clj@@ -6,7 +6,6 @@TODO -- does this belong HERE or in the `permissions` module?"(:require[clojure.set :as set]- [clojure.walk :as walk][metabase.api.common :as api];; legacy usage -- do not use in new code^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.normalize :as mbql.normalize]@@ -32,6 +31,7 @@[metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms][metabase.util.match :as match]+ [metabase.util.performance :as perf][toucan2.core :as t2]))(set! *warn-on-reflection* true)@@ -240,7 +240,7 @@"Return the union of all the `:query-permissions/referenced-card-ids` sets anywhere in the query."[query](let [all-ids (atom #{})]- (walk/postwalk+ (perf/postwalk(fn [form](when (map? form)(when-let [ids (not-empty (:query-permissions/referenced-card-ids form))]diff --git a/src/metabase/query_processor/reducible.clj b/src/metabase/query_processor/reducible.cljindex 020a401042f..86ed5514091 100644--- a/src/metabase/query_processor/reducible.clj+++ b/src/metabase/query_processor/reducible.clj@@ -94,24 +94,22 @@[primary-rf :- ::qp.schema/rfadditional-rfs :- [:sequential ::qp.schema/rf]combine :- ifn?]- (let [additional-accs (volatile! (perf/mapv (fn [rf] (rf))- additional-rfs))]+ (let [additional-rfs (vec additional-rfs)+ ^objects additional-accs (perf/mapa (fn [rf] (rf)) additional-rfs)](fn combine-additional-reducing-fns-rf*([] (primary-rf))([acc]- (let [additional-results (perf/mapv (fn [rf acc]- (rf (unreduced acc)))- additional-rfs- @additional-accs)]- (apply combine acc additional-results)))+ (areduce additional-accs i _ nil+ (let [rf (nth additional-rfs i)+ acc (aget additional-accs i)]+ (aset additional-accs i (rf (unreduced acc)))))+ (apply combine acc (clojure.lang.LazilyPersistentVector/createOwning additional-accs)))([acc x]- (vswap! additional-accs (fn [accs]- (perf/mapv (fn [rf acc]- (if (reduced? acc)- acc- (rf acc x)))- additional-rfs- accs)))+ (areduce additional-accs i _ nil+ (let [rf (nth additional-rfs i)+ acc (aget additional-accs i)]+ (when-not (reduced? acc)+ (aset additional-accs i (rf acc x)))))(primary-rf acc x)))))diff --git a/src/metabase/util/performance.cljc b/src/metabase/util/performance.cljcindex bab45076aa6..2b00550a9c9 100644--- a/src/metabase/util/performance.cljc+++ b/src/metabase/util/performance.cljc@@ -356,19 +356,16 @@(throw (ex-info "Position should be in the range [0; N] of vector." {})))(let [v (vec v)n (count v)]- (if (= pos n)- (conj v el)- (let [i (volatile! -1)]- (persistent! (reduce (fn [acc x]- (vswap! i inc)- (conj! (cond-> acc- (= @i pos) (conj! el))- x))- #?(:clj (if (<= n 32)- (small-transient (inc n) identity)- (transient []))- :cljs (transient []))- v))))))+ (loop [i 0, acc #?(:clj (if (<= n 32)+ (small-transient (inc n) identity)+ (transient []))+ :cljs (transient []))]+ (if (< i pos)+ (recur (inc i) (conj! acc (nth v i)))+ (loop [i pos, acc (conj! acc el)]+ (if (< i n)+ (recur (inc i) (conj! acc (nth v i)))+ (persistent! acc)))))))#?(:clj(defn transpose