The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Compile mbql stages to CTEs for postgres (#81914)
Riley Thompsoncdc7f386afa7
diff --git a/docs/developers-guide/driver-changelog.md b/docs/developers-guide/driver-changelog.mdindex f679c4d54f7..9a20bce4b02 100644--- a/docs/developers-guide/driver-changelog.md+++ b/docs/developers-guide/driver-changelog.md@@ -4,6 +4,12 @@ title: Driver interface changelog# Driver Interface Changelog+## Metabase 0.65.0++- `sql.qp/use-ctes-for-stages?` is a multi-method for drivers to opt-in to compiling multi-stage queries+ with CTEs instead of nested subselects. Drivers should only do this if they satisfy all of the criteria+ in the docstring of this method.+## Metabase 0.64.0- `metabase.driver.sql.normalize/default-schema` now takes the database as well as the driver:diff --git a/src/metabase/driver/postgres.clj b/src/metabase/driver/postgres.cljindex eed5749c4b4..7bfcea3175d 100644--- a/src/metabase/driver/postgres.clj+++ b/src/metabase/driver/postgres.clj@@ -493,6 +493,10 @@[_ json-field-identifier][:length [:cast json-field-identifier :text]])+(defmethod sql.qp/use-ctes-for-stages? :postgres+ [_driver]+ true)+(defmethod sql.qp/add-interval-honeysql-form :postgres[driver hsql-form amount unit](h2x/add-interval-honeysql-form driver hsql-form amount unit))diff --git a/src/metabase/driver/sql/query_processor.clj b/src/metabase/driver/sql/query_processor.cljindex 4a710d92eb8..7eae0abc177 100644--- a/src/metabase/driver/sql/query_processor.clj+++ b/src/metabase/driver/sql/query_processor.clj@@ -2220,6 +2220,32 @@driver-api/add-alias-info:stages))+(defmulti use-ctes-for-stages?+ "Whether to compile the stages of a multi-stage query with CTEs instead of nested subselects, e.g.++ WITH \"__mb_stage_0\" AS (SELECT \"a\", \"b\" FROM \"t\" WHERE \"b\" = 1)+ SELECT \"a\", COUNT(*) FROM \"__mb_stage_0\" AS \"__mb_source\" GROUP BY \"a\"++ instead of++ SELECT \"a\", COUNT(*) FROM (SELECT \"a\", \"b\" FROM \"t\" WHERE \"b\" = 1) AS \"__mb_source\" GROUP BY \"a\"++ Default is `false`. A driver should only opt in if its database accepts a `WITH` clause everywhere Metabase might+ put a compiled query, not just at the top level of a statement:++ * inside the parens of a `JOIN`, since a multi-stage join source compiles to `JOIN (WITH ... SELECT ...) AS j`+ * as the body of another CTE, since a native first stage compiles to `WITH __mb_stage_0 AS (<native SQL>) ...` and+ that native SQL may itself start with `WITH`+ * inside a subquery, e.g. a card referenced from a native query via `{{#123}}` or a metadata probe+ * after `CREATE TABLE ... AS` and `INSERT INTO ...`, for transforms and persisted models"+ {:added "0.65.0", :arglists '([driver])}+ driver/dispatch-on-initialized-driver+ :hierarchy #'driver/hierarchy)++(defmethod use-ctes-for-stages? :sql+ [_driver]+ false)+(defn- desired-col-alias-ident [col](h2x/identifier :field (:lib/desired-column-alias col)))@@ -2251,7 +2277,9 @@(binding [*inner-query* stage](apply-top-level-clauses driver prev-from stage))))-(defn- stages->honeysql [driver stages]+(defn- stages->honeysql-subselects+ "Compile `stages` to HoneySQL with each stage nested as a subselect in the `FROM` of the next one."+ [driver stages](first(reduce(fn [[prev-hsql prev-stage] stage]@@ -2260,6 +2288,42 @@[nil nil]stages)))+(defn- stage-cte-name [stage-idx]+ (str "__mb_stage_" stage-idx))++(defn- cte-stage-source-form+ "The CTE is aliased as `__mb_source` so field refs can compile the same as with nested subselects."+ [driver prev-stage-idx]+ {:from [[(->honeysql driver (h2x/identifier :table-alias (stage-cte-name prev-stage-idx)))+ [(->honeysql driver (h2x/identifier :table-alias source-query-alias))]]]})++(defn- stage-cte+ "Adds the CTE name to the CTE body, e.g. \"SELECT a FROM t\" beocomes\"__mb_stage_N AS (SELECT a FROM t)\".+ Applies the fix for duplicate column names similar to `stage-source-form`."+ [stage-idx hsql stage]+ (let [cte-name (stage-cte-name stage-idx)+ columns-metadata (get-in stage [:lib/stage-metadata :columns])]+ (if (needs-cte-for-duplicate-cols? columns-metadata)+ [[cte-name {:columns (mapv desired-col-alias-ident columns-metadata)}] hsql]+ [cte-name hsql])))++(defn- stages->honeysql-ctes+ "Compile `stages` to a HoneySQL CTE, putting each stage in a CTE that the next stage selects from."+ [driver stages]+ (let [stages (vec stages)+ last-idx (dec (count stages))+ cte-body (fn [idx]+ (let [prev-from (if (zero? idx) {} (cte-stage-source-form driver (dec idx)))]+ (stage->honeysql driver prev-from (stages idx))))+ ctes (mapv #(stage-cte % (cte-body %) (stages %)) (range last-idx))]+ (update (cte-body last-idx) :with #(into ctes %))))++(defn- stages->honeysql [driver stages]+ (if (and (use-ctes-for-stages? driver)+ (> (count stages) 1))+ (stages->honeysql-ctes driver stages)+ (stages->honeysql-subselects driver stages)))+(defmethod join-source :sql[driver {:keys [stages]}](stages->honeysql driver stages))diff --git a/test/metabase/driver/postgres_test.clj b/test/metabase/driver/postgres_test.cljindex 9cd292e9875..ac3eb8c2798 100644--- a/test/metabase/driver/postgres_test.clj+++ b/test/metabase/driver/postgres_test.clj@@ -601,21 +601,22 @@{:database (meta/id):type :query:query {:source-table "card__123"}})]- (is (= ["SELECT"+ (is (= ["WITH \"__mb_stage_0\" AS ("+ " SELECT"+ " (\"json_alias_test\".\"bob\" #>> (array [ ?, ? ] :: text [ ])) :: VARCHAR AS \"json_alias_test\","+ " COUNT(*) AS \"count\""+ " FROM"+ " \"json_alias_test\""+ " GROUP BY"+ " \"json_alias_test\""+ " ORDER BY"+ " \"json_alias_test\" ASC"+ ")"+ "SELECT"" \"__mb_source\".\"json_alias_test\" AS \"json_alias_test\","" \"__mb_source\".\"count\" AS \"count\"""FROM"- " ("- " SELECT"- " (\"json_alias_test\".\"bob\" #>> (array [ ?, ? ] :: text [ ])) :: VARCHAR AS \"json_alias_test\","- " COUNT(*) AS \"count\""- " FROM"- " \"json_alias_test\""- " GROUP BY"- " \"json_alias_test\""- " ORDER BY"- " \"json_alias_test\" ASC"- " ) AS \"__mb_source\""]+ " \"__mb_stage_0\" AS \"__mb_source\""](str/split-lines (driver/prettify-native-form :postgres (:query nested))))))))))(deftest ^:parallel nested-field-pivot-compile-test@@ -649,7 +650,15 @@:show-column-totals true})](qp.store/with-metadata-provider mp(let [sql (:query (qp.compile/compile (nest-for-pivot/wrap-nested-field-breakouts pivot-q)))]- (is (= ["SELECT"+ (is (= ["WITH \"__mb_stage_0\" AS ("+ " SELECT"+ " (\"json_table\".\"payload\" #>> (array [ ? ] :: text [ ])) :: text AS \"category\","+ " \"json_table\".\"region\" AS \"region\","+ " (\"json_table\".\"payload\" #>> (array [ ? ] :: text [ ])) :: text AS \"__mb_pivot_nfc\""+ " FROM"+ " \"json_table\""+ ")"+ "SELECT"" \"__mb_source\".\"__mb_pivot_nfc\" AS \"__mb_pivot_nfc\","" \"__mb_source\".\"region\" AS \"region\","" GROUPING("@@ -658,14 +667,7 @@" ) AS \"pivot-grouping\","" COUNT(*) AS \"count\"""FROM"- " ("- " SELECT"- " (\"json_table\".\"payload\" #>> (array [ ? ] :: text [ ])) :: text AS \"category\","- " \"json_table\".\"region\" AS \"region\","- " (\"json_table\".\"payload\" #>> (array [ ? ] :: text [ ])) :: text AS \"__mb_pivot_nfc\""- " FROM"- " \"json_table\""- " ) AS \"__mb_source\""+ " \"__mb_stage_0\" AS \"__mb_source\"""GROUP BY"" GROUPING SETS ("" ("@@ -1748,6 +1750,70 @@"ORDER BY attempts.date ASC")(some-> (qp.compile/compile query) :query pretty-sql))))))))+(defn- count-by-price-over-1-query+ "Two-stage query: `venues` counted by `price`, then filtered to rows with more than one venue."+ [mp]+ (as-> (lib/query mp (lib.metadata/table mp (mt/id :venues))) $q+ (lib/aggregate $q (lib/count))+ (lib/breakout $q (lib.metadata/field mp (mt/id :venues :price)))+ (lib/append-stage $q)+ (lib/filter $q (lib/> (m/find-first (comp #{"count"} :name) (lib/filterable-columns $q)) 1))))++(deftest ^:parallel multi-stage-query-test+ (testing "a multi-stage query, compiled as chained CTEs on Postgres, runs and returns the right result"+ (mt/test-driver :postgres+ (mt/dataset test-data+ (let [mp (mt/metadata-provider)+ query (as-> (count-by-price-over-1-query mp) $q+ (lib/append-stage $q)+ (lib/aggregate $q (lib/sum (m/find-first (comp #{"count"} :name) (lib/aggregable-columns $q nil)))))]+ (is (= [[100]]+ (mt/formatted-rows [int] (qp/process-query query)))))))))++(deftest ^:parallel multi-stage-join-source-test+ (testing "a multi-stage join source, compiled as a WITH nested inside the JOIN parens on Postgres, runs"+ (mt/test-driver :postgres+ (mt/dataset test-data+ (let [mp (mt/metadata-provider)+ venues (lib.metadata/table mp (mt/id :venues))+ price (lib.metadata/field mp (mt/id :venues :price))+ source (count-by-price-over-1-query mp)+ rhs (m/find-first (comp #{"price"} :name) (lib/returned-columns source))+ join (-> (lib/join-clause source [(lib/= price rhs)])+ (lib/with-join-alias "J")+ (lib/with-join-fields :all))+ query (-> (lib/query mp venues)+ (lib/join join)+ (lib/limit 1))]+ (is (= [[1 "Red Medicine" 4 10.0646 -165.374 3 3 13]]+ (mt/formatted-rows [int str int 4.0 4.0 int int int] (qp/process-query query)))))))))++(deftest ^:parallel native-source-stage-test+ (testing "a native first stage that has its own WITH, compiled as the body of a stage CTE on Postgres, runs"+ (mt/test-driver :postgres+ (mt/dataset test-data+ (let [mp (mt/metadata-provider)+ ;; lib can't see the columns of an unsaved native stage, so the filter uses a literal ref+ cnt [:field {:lib/uuid (str (random-uuid)), :base-type :type/Integer} "cnt"]+ query (-> (lib/native-query mp "WITH v AS (SELECT price, count(*) AS cnt FROM venues GROUP BY price) SELECT * FROM v;")+ (lib/append-stage)+ (lib/filter (lib/> cnt 1))+ (lib/aggregate (lib/count)))]+ (is (= [[4]]+ (mt/formatted-rows [int] (qp/process-query query)))))))))++(deftest ^:parallel multi-stage-card-referenced-from-native-query-test+ (testing "a multi-stage card spliced into a native query via {{#id}}, with its CTEs inside the subquery parens, runs"+ (mt/test-driver :postgres+ (mt/dataset test-data+ (let [mp (mt/metadata-provider)]+ (mt/with-temp [:model/Card card {:dataset_query (count-by-price-over-1-query mp)}]+ (let [tag (format "#%d" (:id card))+ query (-> (lib/native-query mp (format "SELECT SUM(c.count) FROM {{%s}} AS c" tag))+ (lib/with-template-tags {tag {:name tag, :display-name tag, :type :card, :card-id (:id card)}}))]+ (is (= [[100]]+ (mt/formatted-rows [int] (qp/process-query query)))))))))))+(deftest ^:parallel do-not-cast-to-timestamp-if-column-if-timestamp-tz-or-date-test(testing "Don't cast a DATE or TIMESTAMPTZ to TIMESTAMP, it's not necessary (#19816)"(mt/test-driver :postgres