The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Compile MBQL to CTEs for BigQuery, ClickHouse, MySQL, and Vertica (#85187)
Riley Thompsonec51812dccba
diff --git a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/query_processor.clj b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/query_processor.cljindex 6bff57eec54..c962e5982de 100644--- a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/query_processor.clj+++ b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/query_processor.clj@@ -1002,6 +1002,10 @@new-form((get-method ->temporal-type :default) target-type original-form))))+(defmethod sql.qp/use-ctes-for-stages? :bigquery-cloud-sdk+ [_driver]+ true)+(defmethod sql.qp/add-interval-honeysql-form :bigquery-cloud-sdk[_ hsql-form amount unit];; `timestamp_add()` doesn't support month/quarter/year, `add-interval-form` casts it todiff --git a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test.clj b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test.cljindex 9251f256dbe..ecdf999ace2 100644--- a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test.clj+++ b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test.clj@@ -1174,24 +1174,25 @@:limit 2})](mt/with-native-query-testing-context query(is (= (with-test-db-name- {:query ["SELECT"+ {:query ["WITH `__mb_stage_0` AS ("+ " SELECT"+ " DATE_TRUNC("+ " `v4_test_data.checkins`.`date`,"+ " month"+ " ) AS `date`,"+ " COUNT(*) AS `count`"+ " FROM"+ " `v4_test_data.checkins`"+ " GROUP BY"+ " `date`"+ " ORDER BY"+ " `date` ASC"+ ")"+ "SELECT"" `__mb_source`.`count` AS `count`,"" COUNT(*) AS `count_2`""FROM"- " ("- " SELECT"- " DATE_TRUNC("- " `v4_test_data.checkins`.`date`,"- " month"- " ) AS `date`,"- " COUNT(*) AS `count`"- " FROM"- " `v4_test_data.checkins`"- " GROUP BY"- " `date`"- " ORDER BY"- " `date` ASC"- " ) AS `__mb_source`"+ " `__mb_stage_0` AS `__mb_source`""GROUP BY"" `count`""ORDER BY"@@ -1239,15 +1240,16 @@{:aggregation [[:count]]:breakout [[:field "source" {:base-type :type/Text}]]:source-query {:native "select 1 as `val`, '2' as `source`"}})]- (is (= {:query ["SELECT"+ (is (= {:query ["WITH `__mb_stage_0` AS ("+ " select"+ " 1 as `val`,"+ " '2' as `source`"+ ")"+ "SELECT"" `__mb_source`.`source` AS `source`,"" COUNT(*) AS `count`""FROM"- " ("- " select"- " 1 as `val`,"- " '2' as `source`"- " ) AS `__mb_source`"+ " `__mb_stage_0` AS `__mb_source`""GROUP BY"" `source`""ORDER BY"@@ -1331,7 +1333,17 @@;; 4. sum of order total(lib/aggregate (lib/sum orders-total))(lib/limit 3))]- (is (= ["SELECT"+ (is (= ["WITH `__mb_stage_0` AS ("+ " SELECT"+ " TIMESTAMP_TRUNC("+ " `test_data.orders`.`created_at`,"+ " month"+ " ) AS `created_at`,"+ " `test_data.orders`.`total` AS `total`"+ " FROM"+ " `test_data.orders`"+ ")"+ "SELECT"" `__mb_source`.`created_at` AS `created_at`,"" SUM(COUNT(*)) OVER ("" ORDER BY"@@ -1343,16 +1355,7 @@" ) AS `sum`,"" SUM(`__mb_source`.`total`) AS `sum_2`""FROM"- " ("- " SELECT"- " TIMESTAMP_TRUNC("- " `test_data.orders`.`created_at`,"- " month"- " ) AS `created_at`,"- " `test_data.orders`.`total` AS `total`"- " FROM"- " `test_data.orders`"- " ) AS `__mb_source`"+ " `__mb_stage_0` AS `__mb_source`""GROUP BY"" `created_at`""ORDER BY"diff --git a/modules/drivers/clickhouse/src/metabase/driver/clickhouse_qp.clj b/modules/drivers/clickhouse/src/metabase/driver/clickhouse_qp.cljindex 9513ce58bb3..ab3fa4e9439 100644--- a/modules/drivers/clickhouse/src/metabase/driver/clickhouse_qp.clj+++ b/modules/drivers/clickhouse/src/metabase/driver/clickhouse_qp.clj@@ -41,6 +41,10 @@(defmethod sql.qp/quote-style :clickhouse [_] ::clickhouse)+(defmethod sql.qp/use-ctes-for-stages? :clickhouse+ [_driver]+ true)+;; without try, there might be test failures when QP is not yet initialized;; e.g., when a test is preparing the dataset(defn- get-report-timezone-id-safelydiff --git a/modules/drivers/vertica/src/metabase/driver/vertica.clj b/modules/drivers/vertica/src/metabase/driver/vertica.cljindex 597e2f4b7c6..7f0a9dbbd41 100644--- a/modules/drivers/vertica/src/metabase/driver/vertica.clj+++ b/modules/drivers/vertica/src/metabase/driver/vertica.clj@@ -115,6 +115,10 @@[_driver](h2x/with-database-type-info [:current_timestamp [:inline 6]] "timestamptz"))+(defmethod sql.qp/use-ctes-for-stages? :vertica+ [_driver]+ true)+;; Vertica's `GROUPING()` is single-arg only. `GROUPING_ID(a, b, ...)` is its multi-arg counterpart.(defmethod sql.pivot/pivot-grouping-hsql :vertica[_driver exprs]diff --git a/src/metabase/driver/mysql.clj b/src/metabase/driver/mysql.cljindex c5b9deb6f99..1772fd1b1a4 100644--- a/src/metabase/driver/mysql.clj+++ b/src/metabase/driver/mysql.clj@@ -301,6 +301,14 @@[driver hsql-form amount unit](h2x/add-interval-honeysql-form driver hsql-form amount unit))+(defmethod sql.qp/use-ctes-for-stages? :mysql+ [_driver]+ ;; MySQL before 8.0 has no CTEs. All supported MariaDB versions have them.+ (let [db (when (driver-api/initialized?)+ (driver-api/database (driver-api/metadata-provider)))]+ (not (and (mysql? db)+ (some-> db :dbms-version :semantic-version first (< 8))))))+;; now() returns current timestamp in seconds resolution; now(6) returns it in nanosecond resolution(defmethod sql.qp/current-datetime-honeysql-form :mysql[driver]diff --git a/test/metabase/driver/sql/stage_ctes_test.clj b/test/metabase/driver/sql/stage_ctes_test.cljindex 515b7bb21dc..73549bb0e76 100644--- a/test/metabase/driver/sql/stage_ctes_test.clj+++ b/test/metabase/driver/sql/stage_ctes_test.clj@@ -2,7 +2,9 @@"Translation of manual tests done in the UI for CTE compilation into unit tests.1. A multi-stage question with many joins and aggregations2. A native question that selects from 1. with a filter variable- 3. A question that joins 1. with an aggregation and filter"+ 3. A question that joins 1. with an aggregation and filter+ 4. A question on 1. that joins a table, with an aggregation and filter+ 5. A question on a native question that starts with `WITH`"(:require[clojure.test :refer :all][medley.core :as m]@@ -71,33 +73,55 @@(= (:name tag) "sum_limit") (assoc :type :number, :required true, :default "1")))(lib/template-tags query)))))+(defn- filter-and-order-counts+ "Adds a stage with a filter on the count, ordered by birth year, creation year and count, limited to 5 rows."+ [query]+ (let [query (lib/append-stage query)]+ (-> query+ (lib/filter (lib/> (col (lib/filterable-columns query) "count") 11))+ (lib/order-by (col (lib/orderable-columns query) "birth_date"))+ (lib/order-by (col (lib/orderable-columns query) "created_at"))+ (lib/order-by (col (lib/orderable-columns query) "count"))+ (lib/limit 5))))+(defn- query-joining-card- "People joined to Card 1 on source, count by birth year and creation year, then a filter on the count, ordered by- birth year, creation year and count, limited to 5 rows."+ "People joined to Card 1 on source, count by birth year and creation year, then a filter on the count."[mp](let [field #(lib.metadata/field mp (mt/id %1 %2))card (lib.metadata/card mp 1)query (lib/query mp (lib.metadata/table mp (mt/id :people)))query (lib/join query (lib/join-clause card [(lib/= (field :people :source)- (col (lib/join-condition-rhs-columns query card nil nil) "source"))]))- query (-> query- (lib/aggregate (lib/count))- (lib/breakout (lib/with-temporal-bucket (field :people :birth_date) :year))- (lib/breakout (lib/with-temporal-bucket (field :people :created_at) :year))- lib/append-stage)]+ (col (lib/join-condition-rhs-columns query card nil nil) "source"))]))](-> query- (lib/filter (lib/> (col (lib/filterable-columns query) "count") 11))- (lib/order-by (col (lib/orderable-columns query) "birth_date"))- (lib/order-by (col (lib/orderable-columns query) "created_at"))- (lib/order-by (col (lib/orderable-columns query) "count"))- (lib/limit 5))))+ (lib/filter (lib/> (field :people :birth_date) "1970-01-01"))+ (lib/aggregate (lib/count))+ (lib/breakout (lib/with-temporal-bucket (field :people :birth_date) :year))+ (lib/breakout (lib/with-temporal-bucket (field :people :created_at) :year))+ filter-and-order-counts)))++(defn- card-joining-query+ "Card 1 joined to People on source, count by birth year and creation year, then a filter on the count."+ [mp]+ (let [query (lib/query mp (lib.metadata/card mp 1))+ query (lib/join query (lib/join-clause (lib.metadata/table mp (mt/id :people))+ [(lib/= (col (lib/visible-columns query) "source")+ (lib.metadata/field mp (mt/id :people :source)))]))+ people (join-alias query)+ column #(col (lib/breakoutable-columns query) % people)]+ (-> query+ (lib/filter (lib/> (col (lib/filterable-columns query) "birth_date" people) "1970-01-01"))+ (lib/aggregate (lib/count))+ (lib/breakout (lib/with-temporal-bucket (column "birth_date") :year))+ (lib/breakout (lib/with-temporal-bucket (column "created_at") :year))+ filter-and-order-counts)))(deftest ^:parallel multi-stage-questions-test(mt/test-drivers (mt/normal-driver-select {:+fns [#(and (isa? driver/hierarchy % :sql)(sql.qp/use-ctes-for-stages? %))]:+features [:nested-queries :left-join :native-parameter-card-reference]})(let [mp (qp.test-util/metadata-provider-with-cards-for-queries- [(lib/limit (multi-stage-query (mt/metadata-provider)) 100)])+ [(lib/limit (multi-stage-query (mt/metadata-provider)) 100)+ (lib/native-query (mt/metadata-provider) "WITH n AS (SELECT 1 AS x) SELECT x FROM n")])round-sums (partial mt/formatted-rows [str #(u/round-to-decimals 2 (double %))])](testing "multi-stage question with joins"(let [query (multi-stage-query mp)]@@ -121,13 +145,25 @@(is (= [["Affiliate" 281997.3]["Twitter" 302138.79]](sort (round-sums (qp/process-query query))))))))- (testing "question joining the multi-stage question, with more stages after the join"- (let [query (query-joining-card mp)]+ (doseq [[message query-fn] {"question joining the multi-stage question, with more stages after the join"+ query-joining-card+ "question on the multi-stage question joining a table, with more stages after the join"+ card-joining-query}]+ (testing message+ (let [query (query-fn mp)]+ (mt/with-native-query-testing-context query+ (is (= [["1970-01-01T00:00:00Z" "2016-01-01T00:00:00Z" 18]+ ["1970-01-01T00:00:00Z" "2017-01-01T00:00:00Z" 19]+ ["1970-01-01T00:00:00Z" "2018-01-01T00:00:00Z" 21]+ ["1971-01-01T00:00:00Z" "2016-01-01T00:00:00Z" 18]+ ["1971-01-01T00:00:00Z" "2017-01-01T00:00:00Z" 20]]+ (mt/formatted-rows [u.date/temporal-str->iso8601-str u.date/temporal-str->iso8601-str int]+ (qp/process-query query))))))))+ (testing "question on a native question that starts with WITH"+ (let [query (lib/query mp (lib.metadata/card mp 2))](mt/with-native-query-testing-context query- (is (= [["1958-01-01T00:00:00Z" "2017-01-01T00:00:00Z" 13]- ["1958-01-01T00:00:00Z" "2018-01-01T00:00:00Z" 17]- ["1959-01-01T00:00:00Z" "2016-01-01T00:00:00Z" 14]- ["1959-01-01T00:00:00Z" "2017-01-01T00:00:00Z" 19]- ["1959-01-01T00:00:00Z" "2018-01-01T00:00:00Z" 24]]- (mt/formatted-rows [u.date/temporal-str->iso8601-str u.date/temporal-str->iso8601-str int]- (qp/process-query query))))))))))+ (let [query-result (qp/process-query query)]+ (is (= [[1]]+ (mt/formatted-rows [int] query-result)))+ (is (re-find #"WITH.*__mb_stage"+ (-> query-result :data :native_form :query))))))))))