The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Compile snowflake and starburst with CTEs (#83580)
Riley Thompsondc3ed1aef8d4
diff --git a/modules/drivers/athena/src/metabase/driver/athena.clj b/modules/drivers/athena/src/metabase/driver/athena.cljindex a7a9311dd60..ccafb80ce02 100644--- a/modules/drivers/athena/src/metabase/driver/athena.clj+++ b/modules/drivers/athena/src/metabase/driver/athena.clj@@ -170,6 +170,10 @@((get-method sql-jdbc.execute/read-column-thunk [:sql-jdbc Types/OTHER]) driver rs rsmeta i)))+(defmethod sql.qp/use-ctes-for-stages? :athena+ [_driver]+ true)+(defmethod sql.qp/->honeysql [:athena ::sql.qp/cast-to-text][driver [_ _opts expr]](sql.qp/->honeysql driver [::sql.qp/cast {} expr "varchar"]))diff --git a/modules/drivers/databricks/src/metabase/driver/databricks.clj b/modules/drivers/databricks/src/metabase/driver/databricks.cljindex d110231532b..a6aa12cecb2 100644--- a/modules/drivers/databricks/src/metabase/driver/databricks.clj+++ b/modules/drivers/databricks/src/metabase/driver/databricks.clj@@ -115,6 +115,10 @@[schema](str/split schema #"\."))+(defmethod sql.qp/use-ctes-for-stages? :databricks+ [_driver]+ true)+(defmethod sql.qp/->honeysql [:databricks ::h2x/identifier][_driver [tag identifier-type components :as _identifier]](let [components (if (or (and (= identifier-type :table)diff --git a/modules/drivers/snowflake/src/metabase/driver/snowflake.clj b/modules/drivers/snowflake/src/metabase/driver/snowflake.cljindex e9679867f2b..e072138d830 100644--- a/modules/drivers/snowflake/src/metabase/driver/snowflake.clj+++ b/modules/drivers/snowflake/src/metabase/driver/snowflake.clj@@ -636,6 +636,10 @@;; to a different one, so no promotions are possible{})+(defmethod sql.qp/use-ctes-for-stages? :snowflake+ [_driver]+ true)+(defmethod sql.qp/unix-timestamp->honeysql [:snowflake :seconds] [_ _ expr] [:to_timestamp_tz expr])(defmethod sql.qp/unix-timestamp->honeysql [:snowflake :milliseconds] [_ _ expr] [:to_timestamp_tz expr 3])(defmethod sql.qp/unix-timestamp->honeysql [:snowflake :microseconds] [_ _ expr] [:to_timestamp_tz expr 6])diff --git a/modules/drivers/sparksql/src/metabase/driver/sparksql.clj b/modules/drivers/sparksql/src/metabase/driver/sparksql.cljindex 5c95c01bc4f..697d4310074 100644--- a/modules/drivers/sparksql/src/metabase/driver/sparksql.clj+++ b/modules/drivers/sparksql/src/metabase/driver/sparksql.clj@@ -35,6 +35,10 @@"Default alias for all source tables. (Not for source queries; those still use the default SQL QP alias of `source`.)""t1")+(defmethod sql.qp/use-ctes-for-stages? :sparksql+ [_driver]+ true)+(defmethod sql.qp/->honeysql [:sparksql :field][driver [_ {::sql.params.substitution/keys [compiling-field-filter?]} _ :as field-clause]];; use [[source-table-alias]] instead of the usual `schema.table` to qualify fields e.g. `t1.field` instead of thediff --git a/modules/drivers/starburst/src/metabase/driver/starburst.clj b/modules/drivers/starburst/src/metabase/driver/starburst.cljindex d44177b2e57..bf1e40aa19e 100644--- a/modules/drivers/starburst/src/metabase/driver/starburst.clj+++ b/modules/drivers/starburst/src/metabase/driver/starburst.clj@@ -83,6 +83,10 @@:regex/lookaheads-and-lookbehinds false}](defmethod driver/database-supports? [:starburst feature] [_ _ _] supported?))+(defmethod sql.qp/use-ctes-for-stages? :starburst+ [_driver]+ true)+(defn- format-field[name value](if (nil? value)diff --git a/src/metabase/driver/sql/query_processor.clj b/src/metabase/driver/sql/query_processor.cljindex 9ba8deae3a0..ecaab63f850 100644--- a/src/metabase/driver/sql/query_processor.clj+++ b/src/metabase/driver/sql/query_processor.clj@@ -2307,45 +2307,84 @@[nil nil]stages)))-(defn- stage-cte-name [stage-idx]- (str "__mb_stage_" stage-idx))+(def ^:private stage-cte-prefix "__mb_stage_")++(defn- stage-cte-name+ "Name of the CTE for the current nesting level and stage, e.g.+ WITH+ __mb_stage_0 AS (+ WITH+ __mb_stage_0_1 AS (...)+ __mb_stage_1_1 AS (...)+ )+ __mb_stage_1 AS (...)"+ [level stage-idx]+ (cond-> (str stage-cte-prefix stage-idx)+ (pos? level) (str "_" level)))(defn- cte-stage-source-form"The CTE is aliased as `__mb_source` so field refs can compile the same as with nested subselects."- [driver prev-stage-idx]- {:from [[(->honeysql driver (h2x/identifier :table-alias (stage-cte-name prev-stage-idx)))+ [driver level prev-stage-idx]+ {:from [[(->honeysql driver (h2x/identifier :table-alias (stage-cte-name level prev-stage-idx)))[(->honeysql driver (h2x/identifier :table-alias source-query-alias))]]]})(defn- stage-cte- "Adds the CTE name to the CTE body, e.g. \"SELECT a FROM t\" beocomes\"__mb_stage_N AS (SELECT a FROM t)\".+ "Adds the CTE name to the CTE body, e.g. \"SELECT a FROM t\" becomes \"__mb_stage_N AS (SELECT a FROM t)\".Applies the fix for duplicate column names similar to `stage-source-form`."- [stage-idx hsql stage]- (let [cte-name (stage-cte-name stage-idx)+ [level stage-idx hsql stage]+ (let [cte-name (stage-cte-name level stage-idx)columns-metadata (get-in stage [:lib/stage-metadata :columns])](if (needs-cte-for-duplicate-cols? columns-metadata)[[cte-name {:columns (mapv desired-col-alias-ident columns-metadata)}] hsql][cte-name hsql])))(defn- stages->honeysql-ctes- "Compile `stages` to a HoneySQL CTE, putting each stage in a CTE that the next stage selects from."- [driver stages]+ "Compile `stages` to a HoneySQL CTE, putting each stage in a CTE that the next stage selects from.+ `level` is the CTE nesting level, it is added to each stage so that [[join-source]] can nest the joins properly"+ [driver stages level](let [stages (vec stages)last-idx (dec (count stages))- cte-body (fn [idx]- (let [prev-from (if (zero? idx) {} (cte-stage-source-form driver (dec idx)))]- (stage->honeysql driver prev-from (stages idx))))- ctes (mapv #(stage-cte % (cte-body %) (stages %)) (range last-idx))]+ cte-body (fn [idx]+ (let [prev-from (if (zero? idx) {} (cte-stage-source-form driver level (dec idx)))]+ (stage->honeysql driver prev-from (stages idx))))+ ctes (mapv #(stage-cte level % (cte-body %) (stages %)) (range last-idx))](update (cte-body last-idx) :with #(into ctes %))))-(defn- stages->honeysql [driver stages]- (if (and (use-ctes-for-stages? driver)- (> (count stages) 1))- (stages->honeysql-ctes driver stages)- (stages->honeysql-subselects driver stages)))+(defn- stages->honeysql [driver stages level]+ (let [stages (mapv #(assoc % ::cte-level level) stages)]+ (if (and (use-ctes-for-stages? driver)+ (> (count stages) 1))+ (stages->honeysql-ctes driver stages level)+ (stages->honeysql-subselects driver stages))))(defmethod join-source :sql[driver {:keys [stages]}]- (stages->honeysql driver stages))+ (stages->honeysql driver stages (inc (::cte-level *inner-query* 0))))++(defn- all-stages+ "`stages` together with the stages of every join in them"+ [stages]+ (mapcat (fn [stage]+ (cons stage (mapcat (comp all-stages :stages) (:joins stage))))+ stages))++(defn- native-sql-cte-level+ [stage]+ (let [sql (or (:persisted-info/native stage)+ (when (= (:lib/type stage) :mbql.stage/native)+ (:native stage)))]+ (when (string? sql)+ (when-let [levels (seq (for [[_ level] (re-seq (re-pattern (str stage-cte-prefix #"\d+(?:_(\d+))?")) sql)]+ (if level (parse-long level) 0)))]+ (apply max levels)))))++(defn- top-level-cte-level+ "The CTE nesting level for the top-level stages of query. Normally 0, but we check if there are any CTEs+ inside the native SQL of any stage in the query, and set it to one above that to avoid name collisions."+ [stages]+ (if-let [levels (seq (keep native-sql-cte-level (all-stages stages)))]+ (inc (apply max levels))+ 0))(mu/defn mbql->honeysql :- [:or :map [:tuple [:= :inline] :map]]"Build the HoneySQL form we will compile to SQL and execute."@@ -2355,7 +2394,7 @@(binding [driver/*driver* driver](let [stages (preprocess driver query)](log/trace "Compiling MBQL query")- (u/prog1 (stages->honeysql driver stages)+ (u/prog1 (stages->honeysql driver stages (top-level-cte-level stages))(log/debug "Compiled HoneySQL form")(driver-api/debug> (list '🍯 <>)))))(let [metadata-provider (driver-api/metadata-provider)diff --git a/src/metabase/driver/sqlite.clj b/src/metabase/driver/sqlite.cljindex c4f9297605e..ccd12dee74b 100644--- a/src/metabase/driver/sqlite.clj+++ b/src/metabase/driver/sqlite.clj@@ -336,6 +336,10 @@;; See also the [SQLite Date and Time Functions Reference](http://www.sqlite.org/lang_datefunc.html).+(defmethod sql.qp/use-ctes-for-stages? :sqlite+ [_driver]+ true)+;; SQLite stores datetimes as text and the QP wraps datetime literals in `DATETIME(...)` (space-separated),;; so raw column reads (T-separated) don't compare equal to those literals. Wrap only when we can;; confirm the expr is a table-level datetime column (via its `:database-type`) so both sidesdiff --git a/src/metabase/lib/schema.cljc b/src/metabase/lib/schema.cljcindex 43249dc4bd9..4e32c1a1049 100644--- a/src/metabase/lib/schema.cljc+++ b/src/metabase/lib/schema.cljc@@ -102,7 +102,8 @@[:map-of [:ref ::lib.schema.metadata/desired-column-alias] [:ref ::lib.schema.metadata/desired-column-alias]]][:metabase.query-processor.util.add-alias-info/join-alias->escaped{:optional true}- [:map-of [:ref ::join/alias] :string]]])+ [:map-of [:ref ::join/alias] :string]]+ [:metabase.driver.sql.query-processor/cte-level {:optional true} nat-int?]])(mr/def ::stage.native[:anddiff --git a/test/metabase/driver/sqlite_test.clj b/test/metabase/driver/sqlite_test.cljindex 72fdb60fd1c..dde2befdfee 100644--- a/test/metabase/driver/sqlite_test.clj+++ b/test/metabase/driver/sqlite_test.clj@@ -569,11 +569,9 @@(deftest ^:parallel duplicate-identifiers-test(testing "Make sure duplicate identifiers (even with different cases) get unique aliases"(mt/test-driver :sqlite- (is (= '{:select [__mb_source.CATEGORY AS CATEGORY- COUNT (*) AS count]- :from [{:select [products.category || ? AS CATEGORY]- :from [products]}- AS __mb_source]+ (is (= '{:with [__mb_stage_0 AS {:select [products.category || ? AS CATEGORY] :from [products]}]+ :select [__mb_source.CATEGORY AS CATEGORY COUNT (*) AS count]+ :from [__mb_stage_0 AS __mb_source]:group-by [__mb_source.CATEGORY]:order-by [__mb_source.CATEGORY ASC]:limit [1]}diff --git a/test/metabase/lib/serialize_test.cljc b/test/metabase/lib/serialize_test.cljcindex a91bcdda93a..4fc9d4d49ff 100644--- a/test/metabase/lib/serialize_test.cljc+++ b/test/metabase/lib/serialize_test.cljc@@ -204,7 +204,8 @@:metabase.query-processor.middleware.cumulative-aggregations/replaced-indexes #{0}:metabase.query-processor.middleware.add-implicit-joins/reused-join-aliases #{"Cat"}:metabase.query-processor.util.add-alias-info/desired-alias->escaped {"ID" "ID_2"}- :metabase.query-processor.util.add-alias-info/join-alias->escaped {"Cat" "Cat_2"}})+ :metabase.query-processor.util.add-alias-info/join-alias->escaped {"Cat" "Cat_2"}+ :metabase.driver.sql.query-processor/cte-level 1})(def ^:private join-internal-keys"Every internal key middleware adds to a join."