The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Compile Presto queries to CTEs (#84587)
Riley Thompson4e084c403249
diff --git a/.github/workflows/drivers-stress-test.yml b/.github/workflows/drivers-stress-test.ymlindex ebbf1d6dde9..4fd41af3f69 100644--- a/.github/workflows/drivers-stress-test.yml+++ b/.github/workflows/drivers-stress-test.yml@@ -564,7 +564,7 @@ jobs:MB_PRESTO_JDBC_TEST_ADDITIONAL_OPTIONS: 'SSLTrustStorePath=/tmp/cacerts-with-presto-ssl.jks&SSLTrustStorePassword=changeit'services:presto:- image: metabase/presto-mb-ci:latest # version 0.254+ image: metabase/presto-mb-ci:0.295ports:- "8443:8443"env:diff --git a/.github/workflows/drivers.yml b/.github/workflows/drivers.ymlindex 0a257eee640..c7e4cdde918 100644--- a/.github/workflows/drivers.yml+++ b/.github/workflows/drivers.yml@@ -835,7 +835,7 @@ jobs:name: Prestoservices:presto:- image: metabase/presto-mb-ci:latest # version 0.254+ image: metabase/presto-mb-ci:0.295ports:- "8443:8443"env:diff --git a/mage/src/mage/ci_port_map.edn b/mage/src/mage/ci_port_map.ednindex f3ea5213074..5288d3c28d1 100644--- a/mage/src/mage/ci_port_map.edn+++ b/mage/src/mage/ci_port_map.edn@@ -51,7 +51,7 @@"metabase/druid:29.0.1-json-no-ingestion" 8888;; Presto- "metabase/presto-mb-ci:latest" 8443+ "metabase/presto-mb-ci:0.295" 8443;; SparkSQL"metabase/spark:3.2.1" 10000diff --git a/modules/drivers/presto-jdbc/src/metabase/driver/presto_jdbc.clj b/modules/drivers/presto-jdbc/src/metabase/driver/presto_jdbc.cljindex d031c7cb7f7..d7269860913 100644--- a/modules/drivers/presto-jdbc/src/metabase/driver/presto_jdbc.clj+++ b/modules/drivers/presto-jdbc/src/metabase/driver/presto_jdbc.clj@@ -112,6 +112,10 @@(h2x/database-type expr)(h2x/with-database-type-info (h2x/database-type expr)))))+(defmethod sql.qp/use-ctes-for-stages? :presto-jdbc+ [_driver]+ true)+(defmethod sql.qp/add-interval-honeysql-form :presto-jdbc[_driver expr amount unit](date-add unit amount expr))diff --git a/test/metabase/driver/sql/stage_ctes_test.clj b/test/metabase/driver/sql/stage_ctes_test.cljnew file mode 100644index 00000000000..515b7bb21dc--- /dev/null+++ b/test/metabase/driver/sql/stage_ctes_test.clj@@ -0,0 +1,133 @@+(ns ^:mb/driver-tests metabase.driver.sql.stage-ctes-test+ "Translation of manual tests done in the UI for CTE compilation into unit tests.+ 1. A multi-stage question with many joins and aggregations+ 2. A native question that selects from 1. with a filter variable+ 3. A question that joins 1. with an aggregation and filter"+ (:require+ [clojure.test :refer :all]+ [medley.core :as m]+ [metabase.driver :as driver]+ [metabase.driver.sql.query-processor :as sql.qp]+ [metabase.driver.sql.util :as sql.u]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.query-processor.test :as qp]+ [metabase.query-processor.test-util :as qp.test-util]+ [metabase.test :as mt]+ [metabase.util :as u]+ [metabase.util.date-2 :as u.date]))++(defn- col+ ([cols col-name]+ (col cols col-name nil))+ ([cols col-name join-alias]+ (or (m/find-first #(and (= (u/lower-case-en (:name %)) col-name)+ (= (lib/current-join-alias %) join-alias)+ (not= (:lib/source %) :source/implicitly-joinable))+ cols)+ (throw (ex-info "Column not found" {:name col-name, :join-alias join-alias})))))++(defn- join-alias+ [query]+ (:alias (last (lib/joins query))))++(defn- multi-stage-query+ "Total order amount per user and product category, then per user source, sorted by the total:++ stage 0: orders, sum of total by user_id and product_id+ stage 1: join products, sum of the sum by user_id and products.category+ stage 2: join people, sum of the sum by people.source, ordered by the sum"+ [mp]+ (let [field #(lib.metadata/field mp (mt/id %1 %2))+ query (-> (lib/query mp (lib.metadata/table mp (mt/id :orders)))+ (lib/aggregate (lib/sum (field :orders :total)))+ (lib/breakout (field :orders :user_id))+ (lib/breakout (field :orders :product_id))+ lib/append-stage)+ query (lib/join query (lib/join-clause (lib.metadata/table mp (mt/id :products))+ [(lib/= (col (lib/visible-columns query) "product_id")+ (field :products :id))]))+ query (-> query+ (lib/aggregate (lib/sum (col (lib/visible-columns query) "sum")))+ (lib/breakout (col (lib/breakoutable-columns query) "user_id"))+ (lib/breakout (col (lib/breakoutable-columns query) "category" (join-alias query)))+ lib/append-stage)+ query (lib/join query (lib/join-clause (lib.metadata/table mp (mt/id :people))+ [(lib/= (col (lib/visible-columns query) "user_id")+ (field :people :id))]))+ query (-> query+ (lib/aggregate (lib/sum (col (lib/visible-columns query) "sum")))+ (lib/breakout (col (lib/breakoutable-columns query) "source" (join-alias query))))]+ (lib/order-by query (lib/aggregation-ref query 0))))++(defn- native-query-on-card+ "A native query that selects from Card 1 with a required number variable in the `WHERE` clause."+ [mp]+ ;; some databases (e.g. MySQL and Postgres before 16) require an alias for a subquery in FROM+ (let [sql (str "SELECT * FROM {{#1-card-1}} source_card WHERE " (sql.u/quote-name driver/*driver* :field "sum") " < {{sum_limit}}")+ query (lib/native-query mp sql)]+ (lib/with-template-tags query (mapv (fn [tag]+ (cond-> tag+ (= (:name tag) "sum_limit") (assoc :type :number, :required true, :default "1")))+ (lib/template-tags query)))))++(defn- query-joining-card+ "People joined to Card 1 on source, count by birth year and creation year, then a filter on the count, ordered by+ birth year, creation year and count, limited to 5 rows."+ [mp]+ (let [field #(lib.metadata/field mp (mt/id %1 %2))+ card (lib.metadata/card mp 1)+ query (lib/query mp (lib.metadata/table mp (mt/id :people)))+ query (lib/join query (lib/join-clause card [(lib/= (field :people :source)+ (col (lib/join-condition-rhs-columns query card nil nil) "source"))]))+ query (-> query+ (lib/aggregate (lib/count))+ (lib/breakout (lib/with-temporal-bucket (field :people :birth_date) :year))+ (lib/breakout (lib/with-temporal-bucket (field :people :created_at) :year))+ lib/append-stage)]+ (-> query+ (lib/filter (lib/> (col (lib/filterable-columns query) "count") 11))+ (lib/order-by (col (lib/orderable-columns query) "birth_date"))+ (lib/order-by (col (lib/orderable-columns query) "created_at"))+ (lib/order-by (col (lib/orderable-columns query) "count"))+ (lib/limit 5))))++(deftest ^:parallel multi-stage-questions-test+ (mt/test-drivers (mt/normal-driver-select {:+fns [#(and (isa? driver/hierarchy % :sql)+ (sql.qp/use-ctes-for-stages? %))]+ :+features [:nested-queries :left-join :native-parameter-card-reference]})+ (let [mp (qp.test-util/metadata-provider-with-cards-for-queries+ [(lib/limit (multi-stage-query (mt/metadata-provider)) 100)])+ round-sums (partial mt/formatted-rows [str #(u/round-to-decimals 2 (double %))])]+ (testing "multi-stage question with joins"+ (let [query (multi-stage-query mp)]+ (mt/with-native-query-testing-context query+ (let [query-result (qp/process-query query)]+ (is (= [["Affiliate" 281997.3]+ ["Twitter" 302138.79]+ ["Organic" 305583.3]+ ["Google" 306376.03]+ ["Facebook" 314522.28]]+ (round-sums query-result)))+ (is (re-find #"WITH.*__mb_stage"+ (-> query-result :data :native_form :query)))))))+ (testing "native question selecting from the multi-stage question with a variable"+ (let [query (assoc (native-query-on-card mp)+ :parameters [{:type :number/=+ :target [:variable [:template-tag "sum_limit"]]+ :value [305000]}])]+ (mt/with-native-query-testing-context query+ ;; the native query has no ORDER BY, so the order of the rows is not defined+ (is (= [["Affiliate" 281997.3]+ ["Twitter" 302138.79]]+ (sort (round-sums (qp/process-query query))))))))+ (testing "question joining the multi-stage question, with more stages after the join"+ (let [query (query-joining-card mp)]+ (mt/with-native-query-testing-context query+ (is (= [["1958-01-01T00:00:00Z" "2017-01-01T00:00:00Z" 13]+ ["1958-01-01T00:00:00Z" "2018-01-01T00:00:00Z" 17]+ ["1959-01-01T00:00:00Z" "2016-01-01T00:00:00Z" 14]+ ["1959-01-01T00:00:00Z" "2017-01-01T00:00:00Z" 19]+ ["1959-01-01T00:00:00Z" "2018-01-01T00:00:00Z" 24]]+ (mt/formatted-rows [u.date/temporal-str->iso8601-str u.date/temporal-str->iso8601-str int]+ (qp/process-query query))))))))))