Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Compile MBQL to CTEs for BigQuery, ClickHouse, MySQL, and Vertica (#85187)

Riley Thompsonec51812dccba
diff --git a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/query_processor.clj b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/query_processor.clj
index 6bff57eec54..c962e5982de 100644
--- a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/query_processor.clj
+++ b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/query_processor.clj
@@ -1002,6 +1002,10 @@
new-form
((get-method ->temporal-type :default) target-type original-form))))
+(defmethod sql.qp/use-ctes-for-stages? :bigquery-cloud-sdk
+ [_driver]
+ true)
+
(defmethod sql.qp/add-interval-honeysql-form :bigquery-cloud-sdk
[_ hsql-form amount unit]
;; `timestamp_add()` doesn't support month/quarter/year, `add-interval-form` casts it to
diff --git a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test.clj b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test.clj
index 9251f256dbe..ecdf999ace2 100644
--- a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test.clj
+++ b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test.clj
@@ -1174,24 +1174,25 @@
:limit 2})]
(mt/with-native-query-testing-context query
(is (= (with-test-db-name
- {:query ["SELECT"
+ {:query ["WITH `__mb_stage_0` AS ("
+ " SELECT"
+ " DATE_TRUNC("
+ " `v4_test_data.checkins`.`date`,"
+ " month"
+ " ) AS `date`,"
+ " COUNT(*) AS `count`"
+ " FROM"
+ " `v4_test_data.checkins`"
+ " GROUP BY"
+ " `date`"
+ " ORDER BY"
+ " `date` ASC"
+ ")"
+ "SELECT"
" `__mb_source`.`count` AS `count`,"
" COUNT(*) AS `count_2`"
"FROM"
- " ("
- " SELECT"
- " DATE_TRUNC("
- " `v4_test_data.checkins`.`date`,"
- " month"
- " ) AS `date`,"
- " COUNT(*) AS `count`"
- " FROM"
- " `v4_test_data.checkins`"
- " GROUP BY"
- " `date`"
- " ORDER BY"
- " `date` ASC"
- " ) AS `__mb_source`"
+ " `__mb_stage_0` AS `__mb_source`"
"GROUP BY"
" `count`"
"ORDER BY"
@@ -1239,15 +1240,16 @@
{:aggregation [[:count]]
:breakout [[:field "source" {:base-type :type/Text}]]
:source-query {:native "select 1 as `val`, '2' as `source`"}})]
- (is (= {:query ["SELECT"
+ (is (= {:query ["WITH `__mb_stage_0` AS ("
+ " select"
+ " 1 as `val`,"
+ " '2' as `source`"
+ ")"
+ "SELECT"
" `__mb_source`.`source` AS `source`,"
" COUNT(*) AS `count`"
"FROM"
- " ("
- " select"
- " 1 as `val`,"
- " '2' as `source`"
- " ) AS `__mb_source`"
+ " `__mb_stage_0` AS `__mb_source`"
"GROUP BY"
" `source`"
"ORDER BY"
@@ -1331,7 +1333,17 @@
;; 4. sum of order total
(lib/aggregate (lib/sum orders-total))
(lib/limit 3))]
- (is (= ["SELECT"
+ (is (= ["WITH `__mb_stage_0` AS ("
+ " SELECT"
+ " TIMESTAMP_TRUNC("
+ " `test_data.orders`.`created_at`,"
+ " month"
+ " ) AS `created_at`,"
+ " `test_data.orders`.`total` AS `total`"
+ " FROM"
+ " `test_data.orders`"
+ ")"
+ "SELECT"
" `__mb_source`.`created_at` AS `created_at`,"
" SUM(COUNT(*)) OVER ("
" ORDER BY"
@@ -1343,16 +1355,7 @@
" ) AS `sum`,"
" SUM(`__mb_source`.`total`) AS `sum_2`"
"FROM"
- " ("
- " SELECT"
- " TIMESTAMP_TRUNC("
- " `test_data.orders`.`created_at`,"
- " month"
- " ) AS `created_at`,"
- " `test_data.orders`.`total` AS `total`"
- " FROM"
- " `test_data.orders`"
- " ) AS `__mb_source`"
+ " `__mb_stage_0` AS `__mb_source`"
"GROUP BY"
" `created_at`"
"ORDER BY"
diff --git a/modules/drivers/clickhouse/src/metabase/driver/clickhouse_qp.clj b/modules/drivers/clickhouse/src/metabase/driver/clickhouse_qp.clj
index 9513ce58bb3..ab3fa4e9439 100644
--- a/modules/drivers/clickhouse/src/metabase/driver/clickhouse_qp.clj
+++ b/modules/drivers/clickhouse/src/metabase/driver/clickhouse_qp.clj
@@ -41,6 +41,10 @@
(defmethod sql.qp/quote-style :clickhouse [_] ::clickhouse)
+(defmethod sql.qp/use-ctes-for-stages? :clickhouse
+ [_driver]
+ true)
+
;; without try, there might be test failures when QP is not yet initialized
;; e.g., when a test is preparing the dataset
(defn- get-report-timezone-id-safely
diff --git a/modules/drivers/vertica/src/metabase/driver/vertica.clj b/modules/drivers/vertica/src/metabase/driver/vertica.clj
index 597e2f4b7c6..7f0a9dbbd41 100644
--- a/modules/drivers/vertica/src/metabase/driver/vertica.clj
+++ b/modules/drivers/vertica/src/metabase/driver/vertica.clj
@@ -115,6 +115,10 @@
[_driver]
(h2x/with-database-type-info [:current_timestamp [:inline 6]] "timestamptz"))
+(defmethod sql.qp/use-ctes-for-stages? :vertica
+ [_driver]
+ true)
+
;; Vertica's `GROUPING()` is single-arg only. `GROUPING_ID(a, b, ...)` is its multi-arg counterpart.
(defmethod sql.pivot/pivot-grouping-hsql :vertica
[_driver exprs]
diff --git a/src/metabase/driver/mysql.clj b/src/metabase/driver/mysql.clj
index c5b9deb6f99..1772fd1b1a4 100644
--- a/src/metabase/driver/mysql.clj
+++ b/src/metabase/driver/mysql.clj
@@ -301,6 +301,14 @@
[driver hsql-form amount unit]
(h2x/add-interval-honeysql-form driver hsql-form amount unit))
+(defmethod sql.qp/use-ctes-for-stages? :mysql
+ [_driver]
+ ;; MySQL before 8.0 has no CTEs. All supported MariaDB versions have them.
+ (let [db (when (driver-api/initialized?)
+ (driver-api/database (driver-api/metadata-provider)))]
+ (not (and (mysql? db)
+ (some-> db :dbms-version :semantic-version first (< 8))))))
+
;; now() returns current timestamp in seconds resolution; now(6) returns it in nanosecond resolution
(defmethod sql.qp/current-datetime-honeysql-form :mysql
[driver]
diff --git a/test/metabase/driver/sql/stage_ctes_test.clj b/test/metabase/driver/sql/stage_ctes_test.clj
index 515b7bb21dc..73549bb0e76 100644
--- a/test/metabase/driver/sql/stage_ctes_test.clj
+++ b/test/metabase/driver/sql/stage_ctes_test.clj
@@ -2,7 +2,9 @@
"Translation of manual tests done in the UI for CTE compilation into unit tests.
1. A multi-stage question with many joins and aggregations
2. A native question that selects from 1. with a filter variable
- 3. A question that joins 1. with an aggregation and filter"
+ 3. A question that joins 1. with an aggregation and filter
+ 4. A question on 1. that joins a table, with an aggregation and filter
+ 5. A question on a native question that starts with `WITH`"
(:require
[clojure.test :refer :all]
[medley.core :as m]
@@ -71,33 +73,55 @@
(= (:name tag) "sum_limit") (assoc :type :number, :required true, :default "1")))
(lib/template-tags query)))))
+(defn- filter-and-order-counts
+ "Adds a stage with a filter on the count, ordered by birth year, creation year and count, limited to 5 rows."
+ [query]
+ (let [query (lib/append-stage query)]
+ (-> query
+ (lib/filter (lib/> (col (lib/filterable-columns query) "count") 11))
+ (lib/order-by (col (lib/orderable-columns query) "birth_date"))
+ (lib/order-by (col (lib/orderable-columns query) "created_at"))
+ (lib/order-by (col (lib/orderable-columns query) "count"))
+ (lib/limit 5))))
+
(defn- query-joining-card
- "People joined to Card 1 on source, count by birth year and creation year, then a filter on the count, ordered by
- birth year, creation year and count, limited to 5 rows."
+ "People joined to Card 1 on source, count by birth year and creation year, then a filter on the count."
[mp]
(let [field #(lib.metadata/field mp (mt/id %1 %2))
card (lib.metadata/card mp 1)
query (lib/query mp (lib.metadata/table mp (mt/id :people)))
query (lib/join query (lib/join-clause card [(lib/= (field :people :source)
- (col (lib/join-condition-rhs-columns query card nil nil) "source"))]))
- query (-> query
- (lib/aggregate (lib/count))
- (lib/breakout (lib/with-temporal-bucket (field :people :birth_date) :year))
- (lib/breakout (lib/with-temporal-bucket (field :people :created_at) :year))
- lib/append-stage)]
+ (col (lib/join-condition-rhs-columns query card nil nil) "source"))]))]
(-> query
- (lib/filter (lib/> (col (lib/filterable-columns query) "count") 11))
- (lib/order-by (col (lib/orderable-columns query) "birth_date"))
- (lib/order-by (col (lib/orderable-columns query) "created_at"))
- (lib/order-by (col (lib/orderable-columns query) "count"))
- (lib/limit 5))))
+ (lib/filter (lib/> (field :people :birth_date) "1970-01-01"))
+ (lib/aggregate (lib/count))
+ (lib/breakout (lib/with-temporal-bucket (field :people :birth_date) :year))
+ (lib/breakout (lib/with-temporal-bucket (field :people :created_at) :year))
+ filter-and-order-counts)))
+
+(defn- card-joining-query
+ "Card 1 joined to People on source, count by birth year and creation year, then a filter on the count."
+ [mp]
+ (let [query (lib/query mp (lib.metadata/card mp 1))
+ query (lib/join query (lib/join-clause (lib.metadata/table mp (mt/id :people))
+ [(lib/= (col (lib/visible-columns query) "source")
+ (lib.metadata/field mp (mt/id :people :source)))]))
+ people (join-alias query)
+ column #(col (lib/breakoutable-columns query) % people)]
+ (-> query
+ (lib/filter (lib/> (col (lib/filterable-columns query) "birth_date" people) "1970-01-01"))
+ (lib/aggregate (lib/count))
+ (lib/breakout (lib/with-temporal-bucket (column "birth_date") :year))
+ (lib/breakout (lib/with-temporal-bucket (column "created_at") :year))
+ filter-and-order-counts)))
(deftest ^:parallel multi-stage-questions-test
(mt/test-drivers (mt/normal-driver-select {:+fns [#(and (isa? driver/hierarchy % :sql)
(sql.qp/use-ctes-for-stages? %))]
:+features [:nested-queries :left-join :native-parameter-card-reference]})
(let [mp (qp.test-util/metadata-provider-with-cards-for-queries
- [(lib/limit (multi-stage-query (mt/metadata-provider)) 100)])
+ [(lib/limit (multi-stage-query (mt/metadata-provider)) 100)
+ (lib/native-query (mt/metadata-provider) "WITH n AS (SELECT 1 AS x) SELECT x FROM n")])
round-sums (partial mt/formatted-rows [str #(u/round-to-decimals 2 (double %))])]
(testing "multi-stage question with joins"
(let [query (multi-stage-query mp)]
@@ -121,13 +145,25 @@
(is (= [["Affiliate" 281997.3]
["Twitter" 302138.79]]
(sort (round-sums (qp/process-query query))))))))
- (testing "question joining the multi-stage question, with more stages after the join"
- (let [query (query-joining-card mp)]
+ (doseq [[message query-fn] {"question joining the multi-stage question, with more stages after the join"
+ query-joining-card
+ "question on the multi-stage question joining a table, with more stages after the join"
+ card-joining-query}]
+ (testing message
+ (let [query (query-fn mp)]
+ (mt/with-native-query-testing-context query
+ (is (= [["1970-01-01T00:00:00Z" "2016-01-01T00:00:00Z" 18]
+ ["1970-01-01T00:00:00Z" "2017-01-01T00:00:00Z" 19]
+ ["1970-01-01T00:00:00Z" "2018-01-01T00:00:00Z" 21]
+ ["1971-01-01T00:00:00Z" "2016-01-01T00:00:00Z" 18]
+ ["1971-01-01T00:00:00Z" "2017-01-01T00:00:00Z" 20]]
+ (mt/formatted-rows [u.date/temporal-str->iso8601-str u.date/temporal-str->iso8601-str int]
+ (qp/process-query query))))))))
+ (testing "question on a native question that starts with WITH"
+ (let [query (lib/query mp (lib.metadata/card mp 2))]
(mt/with-native-query-testing-context query
- (is (= [["1958-01-01T00:00:00Z" "2017-01-01T00:00:00Z" 13]
- ["1958-01-01T00:00:00Z" "2018-01-01T00:00:00Z" 17]
- ["1959-01-01T00:00:00Z" "2016-01-01T00:00:00Z" 14]
- ["1959-01-01T00:00:00Z" "2017-01-01T00:00:00Z" 19]
- ["1959-01-01T00:00:00Z" "2018-01-01T00:00:00Z" 24]]
- (mt/formatted-rows [u.date/temporal-str->iso8601-str u.date/temporal-str->iso8601-str int]
- (qp/process-query query))))))))))
+ (let [query-result (qp/process-query query)]
+ (is (= [[1]]
+ (mt/formatted-rows [int] query-result)))
+ (is (re-find #"WITH.*__mb_stage"
+ (-> query-result :data :native_form :query))))))))))