Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Compile snowflake and starburst with CTEs (#83580)

Riley Thompsondc3ed1aef8d4
diff --git a/modules/drivers/athena/src/metabase/driver/athena.clj b/modules/drivers/athena/src/metabase/driver/athena.clj
index a7a9311dd60..ccafb80ce02 100644
--- a/modules/drivers/athena/src/metabase/driver/athena.clj
+++ b/modules/drivers/athena/src/metabase/driver/athena.clj
@@ -170,6 +170,10 @@
((get-method sql-jdbc.execute/read-column-thunk [:sql-jdbc Types/OTHER]) driver rs rsmeta i)))
+(defmethod sql.qp/use-ctes-for-stages? :athena
+ [_driver]
+ true)
+
(defmethod sql.qp/->honeysql [:athena ::sql.qp/cast-to-text]
[driver [_ _opts expr]]
(sql.qp/->honeysql driver [::sql.qp/cast {} expr "varchar"]))
diff --git a/modules/drivers/databricks/src/metabase/driver/databricks.clj b/modules/drivers/databricks/src/metabase/driver/databricks.clj
index d110231532b..a6aa12cecb2 100644
--- a/modules/drivers/databricks/src/metabase/driver/databricks.clj
+++ b/modules/drivers/databricks/src/metabase/driver/databricks.clj
@@ -115,6 +115,10 @@
[schema]
(str/split schema #"\."))
+(defmethod sql.qp/use-ctes-for-stages? :databricks
+ [_driver]
+ true)
+
(defmethod sql.qp/->honeysql [:databricks ::h2x/identifier]
[_driver [tag identifier-type components :as _identifier]]
(let [components (if (or (and (= identifier-type :table)
diff --git a/modules/drivers/snowflake/src/metabase/driver/snowflake.clj b/modules/drivers/snowflake/src/metabase/driver/snowflake.clj
index e9679867f2b..e072138d830 100644
--- a/modules/drivers/snowflake/src/metabase/driver/snowflake.clj
+++ b/modules/drivers/snowflake/src/metabase/driver/snowflake.clj
@@ -636,6 +636,10 @@
;; to a different one, so no promotions are possible
{})
+(defmethod sql.qp/use-ctes-for-stages? :snowflake
+ [_driver]
+ true)
+
(defmethod sql.qp/unix-timestamp->honeysql [:snowflake :seconds] [_ _ expr] [:to_timestamp_tz expr])
(defmethod sql.qp/unix-timestamp->honeysql [:snowflake :milliseconds] [_ _ expr] [:to_timestamp_tz expr 3])
(defmethod sql.qp/unix-timestamp->honeysql [:snowflake :microseconds] [_ _ expr] [:to_timestamp_tz expr 6])
diff --git a/modules/drivers/sparksql/src/metabase/driver/sparksql.clj b/modules/drivers/sparksql/src/metabase/driver/sparksql.clj
index 5c95c01bc4f..697d4310074 100644
--- a/modules/drivers/sparksql/src/metabase/driver/sparksql.clj
+++ b/modules/drivers/sparksql/src/metabase/driver/sparksql.clj
@@ -35,6 +35,10 @@
"Default alias for all source tables. (Not for source queries; those still use the default SQL QP alias of `source`.)"
"t1")
+(defmethod sql.qp/use-ctes-for-stages? :sparksql
+ [_driver]
+ true)
+
(defmethod sql.qp/->honeysql [:sparksql :field]
[driver [_ {::sql.params.substitution/keys [compiling-field-filter?]} _ :as field-clause]]
;; use [[source-table-alias]] instead of the usual `schema.table` to qualify fields e.g. `t1.field` instead of the
diff --git a/modules/drivers/starburst/src/metabase/driver/starburst.clj b/modules/drivers/starburst/src/metabase/driver/starburst.clj
index d44177b2e57..bf1e40aa19e 100644
--- a/modules/drivers/starburst/src/metabase/driver/starburst.clj
+++ b/modules/drivers/starburst/src/metabase/driver/starburst.clj
@@ -83,6 +83,10 @@
:regex/lookaheads-and-lookbehinds false}]
(defmethod driver/database-supports? [:starburst feature] [_ _ _] supported?))
+(defmethod sql.qp/use-ctes-for-stages? :starburst
+ [_driver]
+ true)
+
(defn- format-field
[name value]
(if (nil? value)
diff --git a/src/metabase/driver/sql/query_processor.clj b/src/metabase/driver/sql/query_processor.clj
index 9ba8deae3a0..ecaab63f850 100644
--- a/src/metabase/driver/sql/query_processor.clj
+++ b/src/metabase/driver/sql/query_processor.clj
@@ -2307,45 +2307,84 @@
[nil nil]
stages)))
-(defn- stage-cte-name [stage-idx]
- (str "__mb_stage_" stage-idx))
+(def ^:private stage-cte-prefix "__mb_stage_")
+
+(defn- stage-cte-name
+ "Name of the CTE for the current nesting level and stage, e.g.
+ WITH
+ __mb_stage_0 AS (
+ WITH
+ __mb_stage_0_1 AS (...)
+ __mb_stage_1_1 AS (...)
+ )
+ __mb_stage_1 AS (...)"
+ [level stage-idx]
+ (cond-> (str stage-cte-prefix stage-idx)
+ (pos? level) (str "_" level)))
(defn- cte-stage-source-form
"The CTE is aliased as `__mb_source` so field refs can compile the same as with nested subselects."
- [driver prev-stage-idx]
- {:from [[(->honeysql driver (h2x/identifier :table-alias (stage-cte-name prev-stage-idx)))
+ [driver level prev-stage-idx]
+ {:from [[(->honeysql driver (h2x/identifier :table-alias (stage-cte-name level prev-stage-idx)))
[(->honeysql driver (h2x/identifier :table-alias source-query-alias))]]]})
(defn- stage-cte
- "Adds the CTE name to the CTE body, e.g. \"SELECT a FROM t\" beocomes\"__mb_stage_N AS (SELECT a FROM t)\".
+ "Adds the CTE name to the CTE body, e.g. \"SELECT a FROM t\" becomes \"__mb_stage_N AS (SELECT a FROM t)\".
Applies the fix for duplicate column names similar to `stage-source-form`."
- [stage-idx hsql stage]
- (let [cte-name (stage-cte-name stage-idx)
+ [level stage-idx hsql stage]
+ (let [cte-name (stage-cte-name level stage-idx)
columns-metadata (get-in stage [:lib/stage-metadata :columns])]
(if (needs-cte-for-duplicate-cols? columns-metadata)
[[cte-name {:columns (mapv desired-col-alias-ident columns-metadata)}] hsql]
[cte-name hsql])))
(defn- stages->honeysql-ctes
- "Compile `stages` to a HoneySQL CTE, putting each stage in a CTE that the next stage selects from."
- [driver stages]
+ "Compile `stages` to a HoneySQL CTE, putting each stage in a CTE that the next stage selects from.
+ `level` is the CTE nesting level, it is added to each stage so that [[join-source]] can nest the joins properly"
+ [driver stages level]
(let [stages (vec stages)
last-idx (dec (count stages))
- cte-body (fn [idx]
- (let [prev-from (if (zero? idx) {} (cte-stage-source-form driver (dec idx)))]
- (stage->honeysql driver prev-from (stages idx))))
- ctes (mapv #(stage-cte % (cte-body %) (stages %)) (range last-idx))]
+ cte-body (fn [idx]
+ (let [prev-from (if (zero? idx) {} (cte-stage-source-form driver level (dec idx)))]
+ (stage->honeysql driver prev-from (stages idx))))
+ ctes (mapv #(stage-cte level % (cte-body %) (stages %)) (range last-idx))]
(update (cte-body last-idx) :with #(into ctes %))))
-(defn- stages->honeysql [driver stages]
- (if (and (use-ctes-for-stages? driver)
- (> (count stages) 1))
- (stages->honeysql-ctes driver stages)
- (stages->honeysql-subselects driver stages)))
+(defn- stages->honeysql [driver stages level]
+ (let [stages (mapv #(assoc % ::cte-level level) stages)]
+ (if (and (use-ctes-for-stages? driver)
+ (> (count stages) 1))
+ (stages->honeysql-ctes driver stages level)
+ (stages->honeysql-subselects driver stages))))
(defmethod join-source :sql
[driver {:keys [stages]}]
- (stages->honeysql driver stages))
+ (stages->honeysql driver stages (inc (::cte-level *inner-query* 0))))
+
+(defn- all-stages
+ "`stages` together with the stages of every join in them"
+ [stages]
+ (mapcat (fn [stage]
+ (cons stage (mapcat (comp all-stages :stages) (:joins stage))))
+ stages))
+
+(defn- native-sql-cte-level
+ [stage]
+ (let [sql (or (:persisted-info/native stage)
+ (when (= (:lib/type stage) :mbql.stage/native)
+ (:native stage)))]
+ (when (string? sql)
+ (when-let [levels (seq (for [[_ level] (re-seq (re-pattern (str stage-cte-prefix #"\d+(?:_(\d+))?")) sql)]
+ (if level (parse-long level) 0)))]
+ (apply max levels)))))
+
+(defn- top-level-cte-level
+ "The CTE nesting level for the top-level stages of query. Normally 0, but we check if there are any CTEs
+ inside the native SQL of any stage in the query, and set it to one above that to avoid name collisions."
+ [stages]
+ (if-let [levels (seq (keep native-sql-cte-level (all-stages stages)))]
+ (inc (apply max levels))
+ 0))
(mu/defn mbql->honeysql :- [:or :map [:tuple [:= :inline] :map]]
"Build the HoneySQL form we will compile to SQL and execute."
@@ -2355,7 +2394,7 @@
(binding [driver/*driver* driver]
(let [stages (preprocess driver query)]
(log/trace "Compiling MBQL query")
- (u/prog1 (stages->honeysql driver stages)
+ (u/prog1 (stages->honeysql driver stages (top-level-cte-level stages))
(log/debug "Compiled HoneySQL form")
(driver-api/debug> (list '🍯 <>)))))
(let [metadata-provider (driver-api/metadata-provider)
diff --git a/src/metabase/driver/sqlite.clj b/src/metabase/driver/sqlite.clj
index c4f9297605e..ccd12dee74b 100644
--- a/src/metabase/driver/sqlite.clj
+++ b/src/metabase/driver/sqlite.clj
@@ -336,6 +336,10 @@
;; See also the [SQLite Date and Time Functions Reference](http://www.sqlite.org/lang_datefunc.html).
+(defmethod sql.qp/use-ctes-for-stages? :sqlite
+ [_driver]
+ true)
+
;; SQLite stores datetimes as text and the QP wraps datetime literals in `DATETIME(...)` (space-separated),
;; so raw column reads (T-separated) don't compare equal to those literals. Wrap only when we can
;; confirm the expr is a table-level datetime column (via its `:database-type`) so both sides
diff --git a/src/metabase/lib/schema.cljc b/src/metabase/lib/schema.cljc
index 43249dc4bd9..4e32c1a1049 100644
--- a/src/metabase/lib/schema.cljc
+++ b/src/metabase/lib/schema.cljc
@@ -102,7 +102,8 @@
[:map-of [:ref ::lib.schema.metadata/desired-column-alias] [:ref ::lib.schema.metadata/desired-column-alias]]]
[:metabase.query-processor.util.add-alias-info/join-alias->escaped
{:optional true}
- [:map-of [:ref ::join/alias] :string]]])
+ [:map-of [:ref ::join/alias] :string]]
+ [:metabase.driver.sql.query-processor/cte-level {:optional true} nat-int?]])
(mr/def ::stage.native
[:and
diff --git a/test/metabase/driver/sqlite_test.clj b/test/metabase/driver/sqlite_test.clj
index 72fdb60fd1c..dde2befdfee 100644
--- a/test/metabase/driver/sqlite_test.clj
+++ b/test/metabase/driver/sqlite_test.clj
@@ -569,11 +569,9 @@
(deftest ^:parallel duplicate-identifiers-test
(testing "Make sure duplicate identifiers (even with different cases) get unique aliases"
(mt/test-driver :sqlite
- (is (= '{:select [__mb_source.CATEGORY AS CATEGORY
- COUNT (*) AS count]
- :from [{:select [products.category || ? AS CATEGORY]
- :from [products]}
- AS __mb_source]
+ (is (= '{:with [__mb_stage_0 AS {:select [products.category || ? AS CATEGORY] :from [products]}]
+ :select [__mb_source.CATEGORY AS CATEGORY COUNT (*) AS count]
+ :from [__mb_stage_0 AS __mb_source]
:group-by [__mb_source.CATEGORY]
:order-by [__mb_source.CATEGORY ASC]
:limit [1]}
diff --git a/test/metabase/lib/serialize_test.cljc b/test/metabase/lib/serialize_test.cljc
index a91bcdda93a..4fc9d4d49ff 100644
--- a/test/metabase/lib/serialize_test.cljc
+++ b/test/metabase/lib/serialize_test.cljc
@@ -204,7 +204,8 @@
:metabase.query-processor.middleware.cumulative-aggregations/replaced-indexes #{0}
:metabase.query-processor.middleware.add-implicit-joins/reused-join-aliases #{"Cat"}
:metabase.query-processor.util.add-alias-info/desired-alias->escaped {"ID" "ID_2"}
- :metabase.query-processor.util.add-alias-info/join-alias->escaped {"Cat" "Cat_2"}})
+ :metabase.query-processor.util.add-alias-info/join-alias->escaped {"Cat" "Cat_2"}
+ :metabase.driver.sql.query-processor/cte-level 1})
(def ^:private join-internal-keys
"Every internal key middleware adds to a join."