Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Compile mbql stages to CTEs for postgres (#81914)

Riley Thompsoncdc7f386afa7
diff --git a/docs/developers-guide/driver-changelog.md b/docs/developers-guide/driver-changelog.md
index f679c4d54f7..9a20bce4b02 100644
--- a/docs/developers-guide/driver-changelog.md
+++ b/docs/developers-guide/driver-changelog.md
@@ -4,6 +4,12 @@ title: Driver interface changelog
# Driver Interface Changelog
+## Metabase 0.65.0
+
+- `sql.qp/use-ctes-for-stages?` is a multi-method for drivers to opt-in to compiling multi-stage queries
+ with CTEs instead of nested subselects. Drivers should only do this if they satisfy all of the criteria
+ in the docstring of this method.
+
## Metabase 0.64.0
- `metabase.driver.sql.normalize/default-schema` now takes the database as well as the driver:
diff --git a/src/metabase/driver/postgres.clj b/src/metabase/driver/postgres.clj
index eed5749c4b4..7bfcea3175d 100644
--- a/src/metabase/driver/postgres.clj
+++ b/src/metabase/driver/postgres.clj
@@ -493,6 +493,10 @@
[_ json-field-identifier]
[:length [:cast json-field-identifier :text]])
+(defmethod sql.qp/use-ctes-for-stages? :postgres
+ [_driver]
+ true)
+
(defmethod sql.qp/add-interval-honeysql-form :postgres
[driver hsql-form amount unit]
(h2x/add-interval-honeysql-form driver hsql-form amount unit))
diff --git a/src/metabase/driver/sql/query_processor.clj b/src/metabase/driver/sql/query_processor.clj
index 4a710d92eb8..7eae0abc177 100644
--- a/src/metabase/driver/sql/query_processor.clj
+++ b/src/metabase/driver/sql/query_processor.clj
@@ -2220,6 +2220,32 @@
driver-api/add-alias-info
:stages))
+(defmulti use-ctes-for-stages?
+ "Whether to compile the stages of a multi-stage query with CTEs instead of nested subselects, e.g.
+
+ WITH \"__mb_stage_0\" AS (SELECT \"a\", \"b\" FROM \"t\" WHERE \"b\" = 1)
+ SELECT \"a\", COUNT(*) FROM \"__mb_stage_0\" AS \"__mb_source\" GROUP BY \"a\"
+
+ instead of
+
+ SELECT \"a\", COUNT(*) FROM (SELECT \"a\", \"b\" FROM \"t\" WHERE \"b\" = 1) AS \"__mb_source\" GROUP BY \"a\"
+
+ Default is `false`. A driver should only opt in if its database accepts a `WITH` clause everywhere Metabase might
+ put a compiled query, not just at the top level of a statement:
+
+ * inside the parens of a `JOIN`, since a multi-stage join source compiles to `JOIN (WITH ... SELECT ...) AS j`
+ * as the body of another CTE, since a native first stage compiles to `WITH __mb_stage_0 AS (<native SQL>) ...` and
+ that native SQL may itself start with `WITH`
+ * inside a subquery, e.g. a card referenced from a native query via `{{#123}}` or a metadata probe
+ * after `CREATE TABLE ... AS` and `INSERT INTO ...`, for transforms and persisted models"
+ {:added "0.65.0", :arglists '([driver])}
+ driver/dispatch-on-initialized-driver
+ :hierarchy #'driver/hierarchy)
+
+(defmethod use-ctes-for-stages? :sql
+ [_driver]
+ false)
+
(defn- desired-col-alias-ident [col]
(h2x/identifier :field (:lib/desired-column-alias col)))
@@ -2251,7 +2277,9 @@
(binding [*inner-query* stage]
(apply-top-level-clauses driver prev-from stage))))
-(defn- stages->honeysql [driver stages]
+(defn- stages->honeysql-subselects
+ "Compile `stages` to HoneySQL with each stage nested as a subselect in the `FROM` of the next one."
+ [driver stages]
(first
(reduce
(fn [[prev-hsql prev-stage] stage]
@@ -2260,6 +2288,42 @@
[nil nil]
stages)))
+(defn- stage-cte-name [stage-idx]
+ (str "__mb_stage_" stage-idx))
+
+(defn- cte-stage-source-form
+ "The CTE is aliased as `__mb_source` so field refs can compile the same as with nested subselects."
+ [driver prev-stage-idx]
+ {:from [[(->honeysql driver (h2x/identifier :table-alias (stage-cte-name prev-stage-idx)))
+ [(->honeysql driver (h2x/identifier :table-alias source-query-alias))]]]})
+
+(defn- stage-cte
+ "Adds the CTE name to the CTE body, e.g. \"SELECT a FROM t\" beocomes\"__mb_stage_N AS (SELECT a FROM t)\".
+ Applies the fix for duplicate column names similar to `stage-source-form`."
+ [stage-idx hsql stage]
+ (let [cte-name (stage-cte-name stage-idx)
+ columns-metadata (get-in stage [:lib/stage-metadata :columns])]
+ (if (needs-cte-for-duplicate-cols? columns-metadata)
+ [[cte-name {:columns (mapv desired-col-alias-ident columns-metadata)}] hsql]
+ [cte-name hsql])))
+
+(defn- stages->honeysql-ctes
+ "Compile `stages` to a HoneySQL CTE, putting each stage in a CTE that the next stage selects from."
+ [driver stages]
+ (let [stages (vec stages)
+ last-idx (dec (count stages))
+ cte-body (fn [idx]
+ (let [prev-from (if (zero? idx) {} (cte-stage-source-form driver (dec idx)))]
+ (stage->honeysql driver prev-from (stages idx))))
+ ctes (mapv #(stage-cte % (cte-body %) (stages %)) (range last-idx))]
+ (update (cte-body last-idx) :with #(into ctes %))))
+
+(defn- stages->honeysql [driver stages]
+ (if (and (use-ctes-for-stages? driver)
+ (> (count stages) 1))
+ (stages->honeysql-ctes driver stages)
+ (stages->honeysql-subselects driver stages)))
+
(defmethod join-source :sql
[driver {:keys [stages]}]
(stages->honeysql driver stages))
diff --git a/test/metabase/driver/postgres_test.clj b/test/metabase/driver/postgres_test.clj
index 9cd292e9875..ac3eb8c2798 100644
--- a/test/metabase/driver/postgres_test.clj
+++ b/test/metabase/driver/postgres_test.clj
@@ -601,21 +601,22 @@
{:database (meta/id)
:type :query
:query {:source-table "card__123"}})]
- (is (= ["SELECT"
+ (is (= ["WITH \"__mb_stage_0\" AS ("
+ " SELECT"
+ " (\"json_alias_test\".\"bob\" #>> (array [ ?, ? ] :: text [ ])) :: VARCHAR AS \"json_alias_test\","
+ " COUNT(*) AS \"count\""
+ " FROM"
+ " \"json_alias_test\""
+ " GROUP BY"
+ " \"json_alias_test\""
+ " ORDER BY"
+ " \"json_alias_test\" ASC"
+ ")"
+ "SELECT"
" \"__mb_source\".\"json_alias_test\" AS \"json_alias_test\","
" \"__mb_source\".\"count\" AS \"count\""
"FROM"
- " ("
- " SELECT"
- " (\"json_alias_test\".\"bob\" #>> (array [ ?, ? ] :: text [ ])) :: VARCHAR AS \"json_alias_test\","
- " COUNT(*) AS \"count\""
- " FROM"
- " \"json_alias_test\""
- " GROUP BY"
- " \"json_alias_test\""
- " ORDER BY"
- " \"json_alias_test\" ASC"
- " ) AS \"__mb_source\""]
+ " \"__mb_stage_0\" AS \"__mb_source\""]
(str/split-lines (driver/prettify-native-form :postgres (:query nested))))))))))
(deftest ^:parallel nested-field-pivot-compile-test
@@ -649,7 +650,15 @@
:show-column-totals true})]
(qp.store/with-metadata-provider mp
(let [sql (:query (qp.compile/compile (nest-for-pivot/wrap-nested-field-breakouts pivot-q)))]
- (is (= ["SELECT"
+ (is (= ["WITH \"__mb_stage_0\" AS ("
+ " SELECT"
+ " (\"json_table\".\"payload\" #>> (array [ ? ] :: text [ ])) :: text AS \"category\","
+ " \"json_table\".\"region\" AS \"region\","
+ " (\"json_table\".\"payload\" #>> (array [ ? ] :: text [ ])) :: text AS \"__mb_pivot_nfc\""
+ " FROM"
+ " \"json_table\""
+ ")"
+ "SELECT"
" \"__mb_source\".\"__mb_pivot_nfc\" AS \"__mb_pivot_nfc\","
" \"__mb_source\".\"region\" AS \"region\","
" GROUPING("
@@ -658,14 +667,7 @@
" ) AS \"pivot-grouping\","
" COUNT(*) AS \"count\""
"FROM"
- " ("
- " SELECT"
- " (\"json_table\".\"payload\" #>> (array [ ? ] :: text [ ])) :: text AS \"category\","
- " \"json_table\".\"region\" AS \"region\","
- " (\"json_table\".\"payload\" #>> (array [ ? ] :: text [ ])) :: text AS \"__mb_pivot_nfc\""
- " FROM"
- " \"json_table\""
- " ) AS \"__mb_source\""
+ " \"__mb_stage_0\" AS \"__mb_source\""
"GROUP BY"
" GROUPING SETS ("
" ("
@@ -1748,6 +1750,70 @@
"ORDER BY attempts.date ASC")
(some-> (qp.compile/compile query) :query pretty-sql))))))))
+(defn- count-by-price-over-1-query
+ "Two-stage query: `venues` counted by `price`, then filtered to rows with more than one venue."
+ [mp]
+ (as-> (lib/query mp (lib.metadata/table mp (mt/id :venues))) $q
+ (lib/aggregate $q (lib/count))
+ (lib/breakout $q (lib.metadata/field mp (mt/id :venues :price)))
+ (lib/append-stage $q)
+ (lib/filter $q (lib/> (m/find-first (comp #{"count"} :name) (lib/filterable-columns $q)) 1))))
+
+(deftest ^:parallel multi-stage-query-test
+ (testing "a multi-stage query, compiled as chained CTEs on Postgres, runs and returns the right result"
+ (mt/test-driver :postgres
+ (mt/dataset test-data
+ (let [mp (mt/metadata-provider)
+ query (as-> (count-by-price-over-1-query mp) $q
+ (lib/append-stage $q)
+ (lib/aggregate $q (lib/sum (m/find-first (comp #{"count"} :name) (lib/aggregable-columns $q nil)))))]
+ (is (= [[100]]
+ (mt/formatted-rows [int] (qp/process-query query)))))))))
+
+(deftest ^:parallel multi-stage-join-source-test
+ (testing "a multi-stage join source, compiled as a WITH nested inside the JOIN parens on Postgres, runs"
+ (mt/test-driver :postgres
+ (mt/dataset test-data
+ (let [mp (mt/metadata-provider)
+ venues (lib.metadata/table mp (mt/id :venues))
+ price (lib.metadata/field mp (mt/id :venues :price))
+ source (count-by-price-over-1-query mp)
+ rhs (m/find-first (comp #{"price"} :name) (lib/returned-columns source))
+ join (-> (lib/join-clause source [(lib/= price rhs)])
+ (lib/with-join-alias "J")
+ (lib/with-join-fields :all))
+ query (-> (lib/query mp venues)
+ (lib/join join)
+ (lib/limit 1))]
+ (is (= [[1 "Red Medicine" 4 10.0646 -165.374 3 3 13]]
+ (mt/formatted-rows [int str int 4.0 4.0 int int int] (qp/process-query query)))))))))
+
+(deftest ^:parallel native-source-stage-test
+ (testing "a native first stage that has its own WITH, compiled as the body of a stage CTE on Postgres, runs"
+ (mt/test-driver :postgres
+ (mt/dataset test-data
+ (let [mp (mt/metadata-provider)
+ ;; lib can't see the columns of an unsaved native stage, so the filter uses a literal ref
+ cnt [:field {:lib/uuid (str (random-uuid)), :base-type :type/Integer} "cnt"]
+ query (-> (lib/native-query mp "WITH v AS (SELECT price, count(*) AS cnt FROM venues GROUP BY price) SELECT * FROM v;")
+ (lib/append-stage)
+ (lib/filter (lib/> cnt 1))
+ (lib/aggregate (lib/count)))]
+ (is (= [[4]]
+ (mt/formatted-rows [int] (qp/process-query query)))))))))
+
+(deftest ^:parallel multi-stage-card-referenced-from-native-query-test
+ (testing "a multi-stage card spliced into a native query via {{#id}}, with its CTEs inside the subquery parens, runs"
+ (mt/test-driver :postgres
+ (mt/dataset test-data
+ (let [mp (mt/metadata-provider)]
+ (mt/with-temp [:model/Card card {:dataset_query (count-by-price-over-1-query mp)}]
+ (let [tag (format "#%d" (:id card))
+ query (-> (lib/native-query mp (format "SELECT SUM(c.count) FROM {{%s}} AS c" tag))
+ (lib/with-template-tags {tag {:name tag, :display-name tag, :type :card, :card-id (:id card)}}))]
+ (is (= [[100]]
+ (mt/formatted-rows [int] (qp/process-query query)))))))))))
+
(deftest ^:parallel do-not-cast-to-timestamp-if-column-if-timestamp-tz-or-date-test
(testing "Don't cast a DATE or TIMESTAMPTZ to TIMESTAMP, it's not necessary (#19816)"
(mt/test-driver :postgres