Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

[perf] Remove unnecessary MBQL 4 -> 5 -> 4 round trips in SQL drivers (#65509)

Braden Shepherdson9584d61b16bf
diff --git a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/query_processor.clj b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/query_processor.clj
index 3fbc1a40e3d..cd7ad85beec 100644
--- a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/query_processor.clj
+++ b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/query_processor.clj
@@ -981,7 +981,7 @@
(let [parent-method (get-method driver/mbql->native :sql)
compiled (parent-method driver outer-query)]
(assoc compiled
- :table-name (or (when-let [source-table-id (get-in outer-query [:query :source-table])]
+ :table-name (or (when-let [source-table-id (-> outer-query :stages last :source-table)]
(:name (driver-api/table (driver-api/metadata-provider) source-table-id)))
sql.qp/source-query-alias)
:mbql? true)))
diff --git a/modules/drivers/druid/src/metabase/driver/druid/query_processor.clj b/modules/drivers/druid/src/metabase/driver/druid/query_processor.clj
index 7fbdbfb70ea..594f94b242e 100644
--- a/modules/drivers/druid/src/metabase/driver/druid/query_processor.clj
+++ b/modules/drivers/druid/src/metabase/driver/druid/query_processor.clj
@@ -1203,7 +1203,8 @@
"Transpile an MBQL (inner) query into a native form suitable for a Druid DB."
[query]
;; Merge `:settings` into the inner query dict so the QP has access to it
- (let [query (assoc (:query query) :settings (:settings query))]
+ (let [query (driver-api/->legacy-MBQL query)
+ query (assoc (:query query) :settings (:settings query))]
(binding [*query* query
*query-unique-identifier-counter* (atom 0)]
(try
diff --git a/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj b/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj
index f6da77a8382..9cfd104288f 100644
--- a/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj
+++ b/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj
@@ -1779,7 +1779,9 @@ function(bin) {
(defn mbql->native
"Compile an MBQL query."
[query]
- (let [query (update query :query preprocess)]
+ (let [query (-> query
+ driver-api/->legacy-MBQL
+ (update :query preprocess))]
(binding [*query* query
*next-alias-index* (volatile! 0)]
(let [source-table-name (if-let [source-table-id (driver-api/query->source-table-id query)]
diff --git a/src/metabase/driver/sql/query_processor.clj b/src/metabase/driver/sql/query_processor.clj
index 7190696bba9..d4aa4c508dc 100644
--- a/src/metabase/driver/sql/query_processor.clj
+++ b/src/metabase/driver/sql/query_processor.clj
@@ -2011,60 +2011,41 @@
(defmulti preprocess
"Do miscellaneous transformations to the MBQL before compiling the query. These changes are idempotent, so it is safe
to use this function in your own implementations of [[driver/mbql->native]], if you want to apply changes to the
- same version of the query that we will ultimately be compiling."
- {:changelog-test/ignore true, :arglists '([driver inner-query]), :added "0.42.0"}
+ same version of the query that we will ultimately be compiling.
+
+ Wants a `:lib/query` MBQL 5 query as input. Always returns an MBQL 4 **inner query**, for historical reasons."
+ {:changelog-test/ignore true, :arglists '([driver mbql5-query]), :added "0.42.0"}
driver/dispatch-on-initialized-driver
:hierarchy #'driver/hierarchy)
-;;; This is a wrapper
-;;; around [[qp.util.transformations.nest-breakouts/nest-breakouts-in-stages-with-window-aggregation]], which is
-;;; written for pMBQL, so we can use it with a legacy inner query. Once we rework the SQL QP to use pMBQL we can remove
-;;; this.
-(mu/defn- nest-breakouts-in-queries-with-window-fn-aggregations :- driver-api/MBQLQuery
- [inner-query :- driver-api/MBQLQuery]
- (let [metadata-provider (driver-api/metadata-provider)
- database-id (u/the-id (driver-api/database (driver-api/metadata-provider)))]
- (-> (driver-api/query-from-legacy-inner-query metadata-provider database-id inner-query)
- driver-api/nest-breakouts-in-stages-with-window-aggregation
- driver-api/->legacy-MBQL
- :query)))
-
-;;; [[qp.util.transformations.nest-breakouts/nest-breakouts-in-stages-with-window-aggregation]] already does
-;;; basically the same check, this is here mostly to avoid the performance hit of converting to pMBQL and back in
-;;; queries that have no cumulative aggregations at all. Once we convert the SQL QP to pMBQL we can remove this.
-(defn- has-window-function-aggregations? [inner-query]
- (or (driver-api/match (mapcat inner-query [:aggregation :expressions])
- #{:cum-sum :cum-count :offset}
- true)
- (when-let [source-query (:source-query inner-query)]
- (has-window-function-aggregations? source-query))))
-
-(defn- maybe-nest-breakouts-in-queries-with-window-fn-aggregations
- [inner-query]
- (cond-> inner-query
- (has-window-function-aggregations? inner-query)
- nest-breakouts-in-queries-with-window-fn-aggregations))
-
(defmethod preprocess :sql
- [_driver inner-query]
- (-> inner-query
- maybe-nest-breakouts-in-queries-with-window-fn-aggregations
+ [_driver mbql5-query]
+ (-> mbql5-query
+ driver-api/nest-breakouts-in-stages-with-window-aggregation
driver-api/nest-expressions
- driver-api/add-alias-info))
+ driver-api/add-alias-info
+ driver-api/->legacy-MBQL
+ :query))
(mu/defn mbql->honeysql :- [:or :map [:tuple [:= :inline] :map]]
"Build the HoneySQL form we will compile to SQL and execute."
[driver :- :keyword
query :- :map]
(if (:lib/type query)
- (recur driver (driver-api/->legacy-MBQL query))
- (let [{inner-query :query} query]
- (binding [driver/*driver* driver]
- (let [inner-query (preprocess driver inner-query)]
- (log/tracef "Compiling MBQL query\n%s" (u/pprint-to-str 'magenta inner-query))
- (u/prog1 (apply-clauses driver {} inner-query)
- (log/debugf "\nHoneySQL Form: %s\n%s" (u/emoji "🍯") (u/pprint-to-str 'cyan <>))
- (driver-api/debug> (list '🍯 <>))))))))
+ (binding [driver/*driver* driver]
+ (let [inner-query (preprocess driver query)]
+ (log/tracef "Compiling MBQL query\n%s" (u/pprint-to-str 'magenta inner-query))
+ (u/prog1 (apply-clauses driver {} inner-query)
+ (log/debugf "\nHoneySQL Form: %s\n%s" (u/emoji "🍯") (u/pprint-to-str 'cyan <>))
+ (driver-api/debug> (list '🍯 <>)))))
+
+ (let [metadata-provider (driver-api/metadata-provider)
+ database-id (if (:type query)
+ (:database query)
+ (u/the-id (driver-api/database metadata-provider)))
+ inner-query (or (:query query) query)
+ mbql5-query (driver-api/query-from-legacy-inner-query metadata-provider database-id inner-query)]
+ (recur driver mbql5-query))))
;;;; MBQL -> Native
diff --git a/src/metabase/query_processor/compile.clj b/src/metabase/query_processor/compile.clj
index a7ea37ae66b..82de4f63bb8 100644
--- a/src/metabase/query_processor/compile.clj
+++ b/src/metabase/query_processor/compile.clj
@@ -44,7 +44,7 @@
(assert (not (:qp/compiled query)) "This query has already been compiled!")
(if (lib/native-only-query? query)
(set/rename-keys (lib/query-stage query -1) {:native :query})
- (driver/mbql->native driver/*driver* (lib/->legacy-MBQL query))))
+ (driver/mbql->native driver/*driver* query)))
(mu/defn compile-preprocessed :- ::compiled
"Compile an already-preprocessed query, if needed. Returns just the resulting 'inner' native query.
diff --git a/test/metabase/sync/sync_test.clj b/test/metabase/sync/sync_test.clj
index cfffd2b05e5..0963c360c9d 100644
--- a/test/metabase/sync/sync_test.clj
+++ b/test/metabase/sync/sync_test.clj
@@ -97,8 +97,8 @@
true)
(defmethod driver/mbql->native ::sync-test
- [_ query]
- query)
+ [_ _query]
+ {:query "SQL string"})
(defn- ^:dynamic *execute-response*
[query respond]
diff --git a/test/metabase/test/mock/toucanery.clj b/test/metabase/test/mock/toucanery.clj
index 34834d8caf3..9364c164724 100644
--- a/test/metabase/test/mock/toucanery.clj
+++ b/test/metabase/test/mock/toucanery.clj
@@ -118,8 +118,8 @@
{:keypath "movies.description", :value "A cinematic adventure."}]))
(defmethod driver/mbql->native ::toucanery
- [_ query]
- query)
+ [_ _query]
+ {:query "SQL string"})
(defmethod driver/execute-reducible-query ::toucanery
[_ query _ respond]