The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
[perf] Remove unnecessary MBQL 4 -> 5 -> 4 round trips in SQL drivers (#65509)
Braden Shepherdson9584d61b16bf
diff --git a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/query_processor.clj b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/query_processor.cljindex 3fbc1a40e3d..cd7ad85beec 100644--- a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/query_processor.clj+++ b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/query_processor.clj@@ -981,7 +981,7 @@(let [parent-method (get-method driver/mbql->native :sql)compiled (parent-method driver outer-query)](assoc compiled- :table-name (or (when-let [source-table-id (get-in outer-query [:query :source-table])]+ :table-name (or (when-let [source-table-id (-> outer-query :stages last :source-table)](:name (driver-api/table (driver-api/metadata-provider) source-table-id)))sql.qp/source-query-alias):mbql? true)))diff --git a/modules/drivers/druid/src/metabase/driver/druid/query_processor.clj b/modules/drivers/druid/src/metabase/driver/druid/query_processor.cljindex 7fbdbfb70ea..594f94b242e 100644--- a/modules/drivers/druid/src/metabase/driver/druid/query_processor.clj+++ b/modules/drivers/druid/src/metabase/driver/druid/query_processor.clj@@ -1203,7 +1203,8 @@"Transpile an MBQL (inner) query into a native form suitable for a Druid DB."[query];; Merge `:settings` into the inner query dict so the QP has access to it- (let [query (assoc (:query query) :settings (:settings query))]+ (let [query (driver-api/->legacy-MBQL query)+ query (assoc (:query query) :settings (:settings query))](binding [*query* query*query-unique-identifier-counter* (atom 0)](trydiff --git a/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj b/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.cljindex f6da77a8382..9cfd104288f 100644--- a/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj+++ b/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj@@ -1779,7 +1779,9 @@ function(bin) {(defn mbql->native"Compile an MBQL query."[query]- (let [query (update query :query preprocess)]+ (let [query (-> query+ driver-api/->legacy-MBQL+ (update :query preprocess))](binding [*query* query*next-alias-index* (volatile! 0)](let [source-table-name (if-let [source-table-id (driver-api/query->source-table-id query)]diff --git a/src/metabase/driver/sql/query_processor.clj b/src/metabase/driver/sql/query_processor.cljindex 7190696bba9..d4aa4c508dc 100644--- a/src/metabase/driver/sql/query_processor.clj+++ b/src/metabase/driver/sql/query_processor.clj@@ -2011,60 +2011,41 @@(defmulti preprocess"Do miscellaneous transformations to the MBQL before compiling the query. These changes are idempotent, so it is safeto use this function in your own implementations of [[driver/mbql->native]], if you want to apply changes to the- same version of the query that we will ultimately be compiling."- {:changelog-test/ignore true, :arglists '([driver inner-query]), :added "0.42.0"}+ same version of the query that we will ultimately be compiling.++ Wants a `:lib/query` MBQL 5 query as input. Always returns an MBQL 4 **inner query**, for historical reasons."+ {:changelog-test/ignore true, :arglists '([driver mbql5-query]), :added "0.42.0"}driver/dispatch-on-initialized-driver:hierarchy #'driver/hierarchy)-;;; This is a wrapper-;;; around [[qp.util.transformations.nest-breakouts/nest-breakouts-in-stages-with-window-aggregation]], which is-;;; written for pMBQL, so we can use it with a legacy inner query. Once we rework the SQL QP to use pMBQL we can remove-;;; this.-(mu/defn- nest-breakouts-in-queries-with-window-fn-aggregations :- driver-api/MBQLQuery- [inner-query :- driver-api/MBQLQuery]- (let [metadata-provider (driver-api/metadata-provider)- database-id (u/the-id (driver-api/database (driver-api/metadata-provider)))]- (-> (driver-api/query-from-legacy-inner-query metadata-provider database-id inner-query)- driver-api/nest-breakouts-in-stages-with-window-aggregation- driver-api/->legacy-MBQL- :query)))--;;; [[qp.util.transformations.nest-breakouts/nest-breakouts-in-stages-with-window-aggregation]] already does-;;; basically the same check, this is here mostly to avoid the performance hit of converting to pMBQL and back in-;;; queries that have no cumulative aggregations at all. Once we convert the SQL QP to pMBQL we can remove this.-(defn- has-window-function-aggregations? [inner-query]- (or (driver-api/match (mapcat inner-query [:aggregation :expressions])- #{:cum-sum :cum-count :offset}- true)- (when-let [source-query (:source-query inner-query)]- (has-window-function-aggregations? source-query))))--(defn- maybe-nest-breakouts-in-queries-with-window-fn-aggregations- [inner-query]- (cond-> inner-query- (has-window-function-aggregations? inner-query)- nest-breakouts-in-queries-with-window-fn-aggregations))-(defmethod preprocess :sql- [_driver inner-query]- (-> inner-query- maybe-nest-breakouts-in-queries-with-window-fn-aggregations+ [_driver mbql5-query]+ (-> mbql5-query+ driver-api/nest-breakouts-in-stages-with-window-aggregationdriver-api/nest-expressions- driver-api/add-alias-info))+ driver-api/add-alias-info+ driver-api/->legacy-MBQL+ :query))(mu/defn mbql->honeysql :- [:or :map [:tuple [:= :inline] :map]]"Build the HoneySQL form we will compile to SQL and execute."[driver :- :keywordquery :- :map](if (:lib/type query)- (recur driver (driver-api/->legacy-MBQL query))- (let [{inner-query :query} query]- (binding [driver/*driver* driver]- (let [inner-query (preprocess driver inner-query)]- (log/tracef "Compiling MBQL query\n%s" (u/pprint-to-str 'magenta inner-query))- (u/prog1 (apply-clauses driver {} inner-query)- (log/debugf "\nHoneySQL Form: %s\n%s" (u/emoji "π―") (u/pprint-to-str 'cyan <>))- (driver-api/debug> (list 'π― <>))))))))+ (binding [driver/*driver* driver]+ (let [inner-query (preprocess driver query)]+ (log/tracef "Compiling MBQL query\n%s" (u/pprint-to-str 'magenta inner-query))+ (u/prog1 (apply-clauses driver {} inner-query)+ (log/debugf "\nHoneySQL Form: %s\n%s" (u/emoji "π―") (u/pprint-to-str 'cyan <>))+ (driver-api/debug> (list 'π― <>)))))++ (let [metadata-provider (driver-api/metadata-provider)+ database-id (if (:type query)+ (:database query)+ (u/the-id (driver-api/database metadata-provider)))+ inner-query (or (:query query) query)+ mbql5-query (driver-api/query-from-legacy-inner-query metadata-provider database-id inner-query)]+ (recur driver mbql5-query))));;;; MBQL -> Nativediff --git a/src/metabase/query_processor/compile.clj b/src/metabase/query_processor/compile.cljindex a7ea37ae66b..82de4f63bb8 100644--- a/src/metabase/query_processor/compile.clj+++ b/src/metabase/query_processor/compile.clj@@ -44,7 +44,7 @@(assert (not (:qp/compiled query)) "This query has already been compiled!")(if (lib/native-only-query? query)(set/rename-keys (lib/query-stage query -1) {:native :query})- (driver/mbql->native driver/*driver* (lib/->legacy-MBQL query))))+ (driver/mbql->native driver/*driver* query)))(mu/defn compile-preprocessed :- ::compiled"Compile an already-preprocessed query, if needed. Returns just the resulting 'inner' native query.diff --git a/test/metabase/sync/sync_test.clj b/test/metabase/sync/sync_test.cljindex cfffd2b05e5..0963c360c9d 100644--- a/test/metabase/sync/sync_test.clj+++ b/test/metabase/sync/sync_test.clj@@ -97,8 +97,8 @@true)(defmethod driver/mbql->native ::sync-test- [_ query]- query)+ [_ _query]+ {:query "SQL string"})(defn- ^:dynamic *execute-response*[query respond]diff --git a/test/metabase/test/mock/toucanery.clj b/test/metabase/test/mock/toucanery.cljindex 34834d8caf3..9364c164724 100644--- a/test/metabase/test/mock/toucanery.clj+++ b/test/metabase/test/mock/toucanery.clj@@ -118,8 +118,8 @@{:keypath "movies.description", :value "A cinematic adventure."}]))(defmethod driver/mbql->native ::toucanery- [_ query]- query)+ [_ _query]+ {:query "SQL string"})(defmethod driver/execute-reducible-query ::toucanery[_ query _ respond]