The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Enable MBQL5 compilation for drivers (#71439)
Riley Thompson3728dbb5a510
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 291a30b91c5..c85b02c869a 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -1061,6 +1061,7 @@metabase.lib.initmetabase.lib.metadatametabase.lib.metadata.protocols+ metabase.lib.optionsmetabase.lib.schemametabase.lib.schema.actionsmetabase.lib.schema.aggregation@@ -1757,6 +1758,7 @@metabase.query-processor.streaming.interfacemetabase.query-processor.timezonemetabase.query-processor.util+ metabase.query-processor.util.add-alias-infometabase.query-processor.util.persisted-cache}:uses #{analyticsanalytics-interfacediff --git a/docs/developers-guide/driver-changelog.md b/docs/developers-guide/driver-changelog.mdindex 7f8f7044c22..6e8b576b409 100644--- a/docs/developers-guide/driver-changelog.md+++ b/docs/developers-guide/driver-changelog.md@@ -6,6 +6,29 @@ title: Driver interface changelog## Metabase 0.61.0+- Added the following driver multimethods to support MBQL5 compilation migration:+ - `sql.qp/compile-mbql` - Compiles an MBQL inner query to HoneySQL.+ - `sql.qp/mbql-clause-with-opts` - Returns an MBQL clause in the desired MBQL format of the driver.+ - `sql.qp/expression-by-name` - Gets an expression from a query or stage (`*inner-query`) by name.+ - `sql.qp/aggregation-name` - Returns the name of an aggregation clause.+ - `sql.qp/over-order-by->honeysql` - Returns the HoneySQL for an order by clause in the over clause of a window function.+ - `sql.qp/clause-value-idx` - Returns the index of the value in a value clause.+ - `sql.qp/breakout-options-index` - Returns the index of options in a breakout clause.+ - `sql.params.substitution/field->clause` - Returns an MBQL field clause with the given options.+ - `sql.params.substitution/to-clause` - Helper to dispatch to `params.ops/to-clause` or `qp.params.ops/to-clause`.+ - `sql.params.substitution/desugar-filter-clause` - Helper to dispatch to `driver-api/desugar-filter-clause` or `lib/desugar-filter-clause`.+ - `sql.params.substitution/wrap-value-literals-in-mbql` - Helper to dispatch to `driver-api/wrap-value-literals-in-mbql` or `driver-api/wrap-value-literals-in-mbql5`.+ - `sql.params.substitution/date-string->filter` - Helper to dispatch to `params.dates/date-string->filter` or `qp.params.dates/date-string->filter`.++ These methods have implementations for the `:sql` and `:sql-mbql5` drivers. Concrete drivers should *not* need to+ implement these methods. Drivers can opt-in to MBQL5 compilation by adding the `:sql-mbql5` driver as a parent, and updating the `sql.qp/->honeysql`+ methods to handle the clause options argument as the second parameter. See the `:h2` driver in https://github.com/metabase/metabase/pull/71439 for+ an example. Drivers will need to be migrated to work with MBQL5 compilation over the next three releases by v64. After v64 these methods will be deprecated+ in favour of the `:sql-mbql5` implementations once all drivers have been migrated.++- Added a `driver` parameter to `sql.qp/maybe-cast-uuid-for-text-compare`. Any drivers that call this function should+ update it to pass in the `driver` parameter now. An example is in the Snowflake driver's `string-filter` function.+- `driver/field-reference-mlv2`, deprecated in 0.57.0, has now been removed.- `metabase.driver.sql/set-role-statement` has been deprecated in favor ofdiff --git a/enterprise/backend/test/metabase_enterprise/impersonation/driver_test.clj b/enterprise/backend/test/metabase_enterprise/impersonation/driver_test.cljindex 2bd1dc6ee5a..c3610d74377 100644--- a/enterprise/backend/test/metabase_enterprise/impersonation/driver_test.clj+++ b/enterprise/backend/test/metabase_enterprise/impersonation/driver_test.clj@@ -710,7 +710,7 @@(is (thrown-with-msg?clojure.lang.ExceptionInfo;; I've seen different error messages here, not 100% sure why but the important thing is that this fails- #"(?s)SQL compilation error.*(?:(?:operation cannot be performed)|(?:Object.*does not exist or not authorized))"+ #"(?s)(?:SQL compilation error.*(?:(?:operation cannot be performed)|(?:Object.*does not exist or not authorized))|Cannot perform SELECT)"(mt/run-mbql-query venues{:aggregation [[:count]]})))diff --git a/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj b/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.cljindex 6f246fb1729..e2b7c3ced12 100644--- a/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj+++ b/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj@@ -55,9 +55,12 @@(qp.store/with-metadata-provider (mt/id)(sql.qp/->honeysql(or driver/*driver* :h2)- [:field field-id {::add/source-table (mt/id table-key)- ::add/source-alias field-name- ::add/desired-alias field-name}])))))+ (sql.qp/mbql-clause-with-opts driver/*driver*+ :field+ {::add/source-table (mt/id table-key)+ ::add/source-alias field-name+ ::add/desired-alias field-name}+ field-id))))))(defn- venues-category-mbql-gtap-def []{:query (mt/mbql-query venues)diff --git a/modules/drivers/snowflake/src/metabase/driver/snowflake.clj b/modules/drivers/snowflake/src/metabase/driver/snowflake.cljindex ae9db9546c9..dbb2af621ea 100644--- a/modules/drivers/snowflake/src/metabase/driver/snowflake.clj+++ b/modules/drivers/snowflake/src/metabase/driver/snowflake.clj@@ -1038,7 +1038,7 @@(defn- string-filter[driver str-filter field arg {:keys [case-sensitive] :or {case-sensitive true} :as options}]- (let [casted-field (sql.qp/->honeysql driver (sql.qp/maybe-cast-uuid-for-text-compare field))]+ (let [casted-field (sql.qp/->honeysql driver (sql.qp/maybe-cast-uuid-for-text-compare driver field))][str-filter(if case-sensitive casted-field [:lower casted-field])(get-string-filter-arg driver arg options)]))diff --git a/src/metabase/driver/h2.clj b/src/metabase/driver/h2.cljindex 77a6d8ee07a..d290d6466d1 100644--- a/src/metabase/driver/h2.clj+++ b/src/metabase/driver/h2.clj@@ -44,6 +44,12 @@(driver/register! :h2, :parent #{:sql-jdbc ::like-escape-char-built-in/like-escape-char-built-in})+;; h2 can be used to generate mbql5 natively, but this is not the default yet.+;; we need to gather more data from the experiment (see query-processor/mbql->honeysql)+;; before we can switch this over. once that happens, make regular :h2 have+;; :sql-mbql5 as a parent and move the :h2-mbql5 methods below to just :h2.+(driver/register! :h2-mbql5, :parent #{:h2 :sql-mbql5})+;;; this will prevent the H2 driver from showing up in the list of options when adding a new Database.(defmethod driver/superseded-by :h2 [_driver] :deprecated)@@ -94,6 +100,10 @@[driver [_ arg pattern]][:regexp_substr (sql.qp/->honeysql driver arg) (sql.qp/->honeysql driver pattern)])+(defmethod sql.qp/->honeysql [:h2-mbql5 :regex-match-first]+ [driver [_ _opts arg pattern]]+ [:regexp_substr (sql.qp/->honeysql driver arg) (sql.qp/->honeysql driver pattern)])+(defmethod driver/connection-properties :h2[_](->>@@ -419,6 +429,10 @@[driver [_ field]][:log10 (sql.qp/->honeysql driver field)])+(defmethod sql.qp/->honeysql [:h2-mbql5 :log]+ [driver [_ _opts field]]+ [:log10 (sql.qp/->honeysql driver field)])+(defmethod sql.qp/->honeysql [:h2 ::sql.qp/expression-literal-text-value][driver [_ value]];; A literal text value gets compiled to a parameter placeholder like "?". H2 attempts to compile the prepared@@ -430,6 +444,17 @@(->> (sql.qp/->honeysql driver value)(h2x/cast :text)))+(defmethod sql.qp/->honeysql [:h2-mbql5 ::sql.qp/expression-literal-text-value]+ [driver [_ _opts value]]+ ;; A literal text value gets compiled to a parameter placeholder like "?". H2 attempts to compile the prepared+ ;; statement immediately, presumably before the types of the params are known, and sometimes raises an "Unknown+ ;; data type" error if it can't deduce the type. The recommended workaround is to insert an explicit CAST.+ ;;+ ;; https://linear.app/metabase/issue/QUE-726/+ ;; https://github.com/h2database/h2database/issues/1383+ (->> (sql.qp/->honeysql driver value)+ (h2x/cast :text)))+(defn- datediff"Like H2's `datediff` function but accounts for timestamps with time zones."[unit x y]diff --git a/src/metabase/driver/sql/parameters/substitution.clj b/src/metabase/driver/sql/parameters/substitution.cljindex 6f38a0cf3c3..30af4f42a00 100644--- a/src/metabase/driver/sql/parameters/substitution.clj+++ b/src/metabase/driver/sql/parameters/substitution.clj@@ -9,6 +9,7 @@(:refer-clojure :exclude [not-empty mapv])(:require[clojure.string :as str]+ [medley.core :as m][metabase.driver :as driver][metabase.driver-api.core :as driver-api]^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.driver.common.parameters :as params]@@ -265,8 +266,14 @@{:replacement-snippet snippet:prepared-statement-args args}))-(mu/defn- field->clause :- driver-api/mbql.schema.field- [field other-opts]+(defmulti field->clause+ "Returns an MBQL field clause with the given options."+ {:added "0.61.0" :arglists '([driver field other-opts])}+ driver/dispatch-on-initialized-driver+ :hierarchy #'driver/hierarchy)++(mu/defmethod field->clause :default :- driver-api/mbql.schema.field+ [_driver field other-opts](driver-api/normalizedriver-api/mbql.schema.field[:field@@ -276,7 +283,7 @@::compiling-field-filter? true}other-opts)]))-(mu/defn- field->field-filter-clause :- driver-api/mbql.schema.field+(mu/defn- field->field-filter-clause :- [:or driver-api/mbql.schema.field :mbql.clause/field][driver :- :keywordfield :- driver-api/schema.metadata.columnparam-type :- driver-api/schema.parameter.type@@ -287,7 +294,7 @@;; middleware would work either because we don't know what Field this parameter actually refers to until we resolve;; the parameter. There's probably _some_ way to structure things that would make this "duplicate" call unneeded, but;; I haven't figured out what that is yet- (field->clause field {:temporal-unit (align-temporal-unit-with-param-type-and-value driver field param-type value)}))+ (field->clause driver field (m/assoc-some {} :temporal-unit (align-temporal-unit-with-param-type-and-value driver field param-type value))))(mu/defn- field->identifier :- driver-api/schema.common.non-blank-string"Return an appropriate snippet to represent this `field` in SQL given its param type.@@ -310,6 +317,45 @@params/map->DateTimeRange->datetime-replacement-snippet-info)))+(defmulti to-clause+ "Helper to dispatch to `params.ops/to-clause` for `:sql` or `qp.params.ops/to-clause` for `:sql-mbql5`."+ {:added "0.61.0" :arglists '([driver param])}+ driver/dispatch-on-initialized-driver+ :hierarchy #'driver/hierarchy)++(defmethod to-clause :default [_driver param] (params.ops/to-clause param))++(defmulti desugar-filter-clause+ "Helper to dispatch to `driver-api/desugar-filter-clause` for `:sql` or `lib/desugar-filter-clause` for `:sql-mbql5`."+ {:added "0.61.0" :arglists '([driver filter-clause])}+ driver/dispatch-on-initialized-driver+ :hierarchy #'driver/hierarchy)++(defmethod desugar-filter-clause :default [_driver filter-clause]+ #_{:clj-kondo/ignore [:deprecated-var]}+ (driver-api/desugar-filter-clause filter-clause))++(defmulti wrap-value-literals-in-mbql+ "Helper to dispatch to `driver-api/wrap-value-literals-in-mbql` for `:sql`+ or `driver-api/wrap-value-literals-in-mbql5` for `:sql-mbql5`."+ {:added "0.61.0" :arglists '([driver mbql])}+ driver/dispatch-on-initialized-driver+ :hierarchy #'driver/hierarchy)++(defmethod wrap-value-literals-in-mbql :default+ [_driver mbql]+ #_{:clj-kondo/ignore [:deprecated-var]}+ (driver-api/wrap-value-literals-in-mbql mbql))++(defmulti date-string->filter+ "Helper to dispatch to `params.dates/date-string->filter` for `:sql`+ or `qp.params.dates/date-string->filter` for `:sql-mbql5`."+ {:added "0.61.0" :arglists '([driver date-string field])}+ driver/dispatch-on-initialized-driver+ :hierarchy #'driver/hierarchy)++(defmethod date-string->filter :default [_driver date-string field] (params.dates/date-string->filter date-string field))+(mu/defn- field-filter->replacement-snippet-info :- ParamSnippetInfo"Return `[replacement-snippet & prepared-statement-args]` appropriate for a field filter parameter."[driver {{param-type :type, value :value, :as params} :value, field :field, :as field-filter}]@@ -323,18 +369,18 @@(params.ops/operator? param-type)#_{:clj-kondo/ignore [:deprecated-var]}(->> (assoc params :target [:dimension (field->field-filter-clause driver field param-type value)])- params.ops/to-clause- driver-api/desugar-filter-clause- driver-api/wrap-value-literals-in-mbql+ (to-clause driver)+ (desugar-filter-clause driver)+ (wrap-value-literals-in-mbql driver)->honeysql(honeysql->replacement-snippet-info driver))(params.dates/exclusion-date-type param-type value)(let [field-clause (field->field-filter-clause driver field param-type value)]#_{:clj-kondo/ignore [:deprecated-var]}- (->> (params.dates/date-string->filter value field-clause)- driver-api/desugar-filter-clause- driver-api/wrap-value-literals-in-mbql+ (->> (date-string->filter driver value field-clause)+ (desugar-filter-clause driver)+ (wrap-value-literals-in-mbql driver)->honeysql(honeysql->replacement-snippet-info driver)))@@ -361,7 +407,7 @@[driver field alias replacement-snippet-info](if (str/blank? alias)replacement-snippet-info- (let [[old-name] (->> (field->clause field nil)+ (let [[old-name] (->> (field->clause driver field nil)(sql.qp/->honeysql driver)(sql.qp/format-honeysql driver))](update replacement-snippet-info :replacement-snippet str/replace old-name alias))))@@ -403,12 +449,10 @@(defmethod ->replacement-snippet-info [:sql TemporalUnit][driver {:keys [value field alias]}]- (let [replacement-snippet-info- (honeysql->replacement-snippet-info- driver- (sql.qp/->honeysql driver- (field->clause field (when (not= value params/no-value)- {:temporal-unit (keyword value)}))))]+ (let [replacement-snippet-info (->> (field->clause driver field (when (not= value params/no-value)+ {:temporal-unit (keyword value)}))+ (sql.qp/->honeysql driver)+ (honeysql->replacement-snippet-info driver))](replace-alias driver field alias replacement-snippet-info)))(defmethod ->replacement-snippet-info [:sql ReferencedTableQuery]diff --git a/src/metabase/driver/sql/query_processor.clj b/src/metabase/driver/sql/query_processor.cljindex ef18467cf64..f257af50ea3 100644--- a/src/metabase/driver/sql/query_processor.clj+++ b/src/metabase/driver/sql/query_processor.clj@@ -14,6 +14,7 @@[metabase.driver.sql.query-processor.deprecated :as sql.qp.deprecated][metabase.util :as u][metabase.util.date-2 :as u.date]+ [metabase.util.experiment :as experiment][metabase.util.honey-sql-2 :as h2x][metabase.util.i18n :refer [tru]][metabase.util.log :as log]@@ -300,6 +301,21 @@(sql/register-fn! ::compiled #'format-compiled)+(defmulti mbql-clause-with-opts+ "Returns an MBQL clause in the desired MBQL format of the driver."+ {:added "0.61.0" :arglists '([driver tag opts & args])}+ driver/dispatch-on-initialized-driver+ :hierarchy #'driver/hierarchy)++(defmethod mbql-clause-with-opts :sql+ [_driver tag opts & args]+ (cond-> (into [tag] args) opts (conj opts)))++(defn mbql-clause+ "Wrapper around `mbql-clause-with-opts` for clauses that don't have or need options."+ [driver tag & args]+ (apply mbql-clause-with-opts driver tag nil args))+(defmulti current-datetime-honeysql-form"HoneySQL form that should be used to get the current `datetime` (or equivalent). Defaults to `:%now`. Should ideallyinclude the database type info on the form (ex: via [[h2x/with-type-info]])."@@ -332,7 +348,7 @@(defmethod date [:sql :week-of-year][driver _ expr];; Some DBs truncate when doing integer division, therefore force float arithmetic- (->honeysql driver [:ceil (compiled (h2x// (date driver :day-of-year (date driver :week expr)) 7.0))]))+ (->honeysql driver (mbql-clause driver :ceil (compiled (h2x// (date driver :day-of-year (date driver :week expr)) 7.0)))))(defmethod date [:sql :month-of-year] [_driver _ expr] (h2x/month expr))(defmethod date [:sql :quarter-of-year] [_driver _ expr] (h2x/quarter expr))@@ -394,7 +410,7 @@(days-till-start-of-first-full-week driver honeysql-expr))total-full-week-days (h2x/- (date driver :day-of-year honeysql-expr)days-till-start-of-first-full-week)- total-full-weeks (->honeysql driver [:ceil (compiled (h2x// total-full-week-days 7.0))])]+ total-full-weeks (->honeysql driver (mbql-clause driver :ceil (compiled (h2x// total-full-week-days 7.0))))](->integer driver (h2x/+ 1 total-full-weeks))));; ISO8501 consider the first week of the year is the week that contains the 1st Thursday and week starts on Monday.@@ -700,7 +716,7 @@:else value)(->honeysql driver value))))-(defn- literal-text-value?+(defn- literal-text-value?*[[_ value {base-type :base_type effective-type :effective_type} :as clause]](and (driver-api/is-clause? :value clause)(string? value)@@ -710,6 +726,24 @@(isa? (or effective-type base-type):type/Text))))+(defn- literal-text-value?+ [clause]+ (literal-text-value?*+ (driver-api/match-one clause+ [tag (opts :guard :lib/uuid) value] ;; mbql5+ [tag value {:base_type (:base-type opts) :effective_type (:effective-type opts)}]+ _ clause)))++(defmulti expression-by-name+ "Gets an expression from a query or stage (`*inner-query`) by name."+ {:added "0.61.0" :arglists '([driver inner-query expression-name])}+ driver/dispatch-on-initialized-driver+ :hierarchy #'driver/hierarchy)++(defmethod expression-by-name :sql+ [_driver inner-query expression-name]+ (driver-api/expression-with-name inner-query expression-name))+(defn apply-temporal-bucketing"Apply temporal bucketing for the `:temporal-unit` in the options of a `:field` clause; return a new HoneySQL form thatbuckets `honeysql-form` appropriately."@@ -720,13 +754,13 @@[driver [_ expression-name opts :as _clause]](let [source-table (get opts driver-api/qp.add.source-table)source-alias (get opts driver-api/qp.add.source-alias)- expression-definition (driver-api/expression-with-name *inner-query* expression-name)]+ expression-definition (expression-by-name driver *inner-query* expression-name)](cond->>(->honeysql driver (cond (= source-table driver-api/qp.add.source)(apply h2x/identifier :field source-query-alias source-alias)(literal-text-value? expression-definition)- [::expression-literal-text-value expression-definition]+ (mbql-clause driver ::expression-literal-text-value expression-definition);; Handle raw string literals (not wrapped in :value) - needed for;; expression definitions that are just string literals, e.g. from@@ -734,8 +768,8 @@;; the string becomes a parameter placeholder without type info,;; which some databases (like H2) can't handle.(string? expression-definition)- [::expression-literal-text-value- [:value expression-definition {:base_type :type/Text}]]+ (mbql-clause driver ::expression-literal-text-value+ (mbql-clause-with-opts driver :value {:base_type :type/Text} expression-definition)):elseexpression-definition))@@ -931,7 +965,7 @@;; preserve metadata attached to the original field clause, for example BigQuery temporal type information.identifier (-> (apply h2x/identifier :field(concat source-table-aliases- (->honeysql driver [::nfc-path source-nfc-path])+ (->honeysql driver (mbql-clause driver ::nfc-path source-nfc-path))parent-names[source-alias]))(with-meta (meta field-clause)))@@ -1037,41 +1071,64 @@(or (clause-pred (first form))(m/find-first (partial contains-clause? clause-pred) (rest form))))))+(defmulti over-order-by->honeysql+ "Returns the HoneySQL for an order by clause in the over clause of a window function."+ {:added "0.61.0" :arglists '([driver aggregations order-by])}+ driver/dispatch-on-initialized-driver+ :hierarchy #'driver/hierarchy)++(defmethod over-order-by->honeysql :sql+ [driver aggregations [direction expr]]+ (if (aggregation? expr)+ (let [[_aggregation index] expr+ agg (unwrap-aggregation-option (aggregations index))]+ (when-not (contains-clause? #{:cum-count :cum-sum :offset} agg)+ [(->honeysql driver agg) direction]))+ [(->honeysql driver expr) direction]))+(defn- over-order-bys"Returns a vector containing the `aggregations` specified by `order-bys` compiled tohoneysql expressions for `driver` suitable for ordering in the over clause of a window function."[driver aggregations order-bys](let [aggregations (vec aggregations)](into []- (keep (fn [[direction expr]]- (if (aggregation? expr)- (let [[_aggregation index] expr- agg (unwrap-aggregation-option (aggregations index))]- (when-not (contains-clause? #{:cum-count :cum-sum :offset} agg)- [(->honeysql driver agg) direction]))- [(->honeysql driver expr) direction])))+ (keep (fn [order-by]+ (over-order-by->honeysql driver aggregations order-by)))order-bys)))+(defn- remapped-order-by? [order-by]+ (driver-api/qp.util.transformations.nest-breakouts.externally-remapped-field+ (driver-api/match-one order-by+ [_dir (_opts :guard :lib/uuid) [_ (opts :guard :lib/uuid) _name]] opts ;; mbql5+ [_dir [_ _name opts]] opts)))++(defn- remapped-breakout? [breakout]+ (driver-api/qp.util.transformations.nest-breakouts.externally-remapped-field+ (driver-api/match-one breakout+ [_ (opts :guard :lib/uuid) _name] opts ;; mbql5+ [_ _name opts] opts)))++(defmulti breakout-options-index+ "Returns the index of options in a breakout clause."+ {:added "0.61.0", :arglists '([driver])}+ driver/dispatch-on-initialized-driver+ :hierarchy #'driver/hierarchy)++(defmethod breakout-options-index :sql [_driver] 2)+(defn- window-aggregation-over-expr-for-query-with-breakouts"Order by the first breakout, then partition by all the other ones. See #42003 andhttps://metaboat.slack.com/archives/C05MPF0TM3L/p1714084449574689 for more info."[driver inner-query]- (let [breakouts (into []- (remove- (comp driver-api/qp.util.transformations.nest-breakouts.externally-remapped-field- #(nth % 2)))- (:breakout inner-query))+ (let [breakouts (into [] (remove remapped-breakout?) (:breakout inner-query))group-bys (:group-by (apply-top-level-clause driver :breakout {} inner-query))- finest-temp-breakout (driver-api/finest-temporal-breakout-index breakouts 2)+ opts-idx (breakout-options-index driver)+ finest-temp-breakout (driver-api/finest-temporal-breakout-index breakouts opts-idx)partition-exprs (when (> (count breakouts) 1)(if finest-temp-breakout(m/remove-nth finest-temp-breakout group-bys)(butlast group-bys)))- order-bys (remove- (comp driver-api/qp.util.transformations.nest-breakouts.externally-remapped-field- #(nth % 2)- second)- (:order-by inner-query))+ order-bys (remove remapped-order-by? (:order-by inner-query))order-bys (over-order-bys driver(:aggregation inner-query)order-bys)]@@ -1156,7 +1213,7 @@[driver [_cum-count expr-or-nil]];; a cumulative count with no breakouts doesn't really mean anything, just compile it as a normal count.(if (empty? (:breakout *inner-query*))- (->honeysql driver [:count expr-or-nil])+ (->honeysql driver (mbql-clause driver :count expr-or-nil))(cumulative-aggregation-over-rowsdriver[:sum (if expr-or-nil@@ -1178,7 +1235,7 @@[driver [_cum-sum expr]];; a cumulative sum with no breakouts doesn't really mean anything, just compile it as a normal sum.(if (empty? (:breakout *inner-query*))- (->honeysql driver [:sum expr])+ (->honeysql driver (mbql-clause driver :sum expr))(cumulative-aggregation-over-rowsdriver[:sum [:sum (->honeysql driver expr)]])))@@ -1198,6 +1255,11 @@(defn- interval? [expr](driver-api/is-clause? :interval expr))+(defn- normalize-interval [interval]+ (driver-api/match-one interval+ [tag (_opts :guard :lib/uuid) amount unit] [tag amount unit] ;; mbql5+ _ interval))+(defmethod ->honeysql [:sql :+][driver [_ & args]](if (some interval? args)@@ -1205,7 +1267,7 @@(reduce (fn [hsql-form [_ amount unit]](add-interval-honeysql-form driver hsql-form amount unit))(->honeysql driver field)- intervals)+ (map normalize-interval intervals))(throw (ex-info "Summing intervals is not supported" {:args args})))(into [:+](map (partial ->honeysql driver))@@ -1227,7 +1289,7 @@;; We are adding negative amount. Inspired by `->honeysql [:sql :datetime-subtract]`.(add-interval-honeysql-form driver hsql-form (- amount) unit))(->honeysql driver first-arg)- other-args)+ (map normalize-interval other-args))(into [:-](map (partial ->honeysql driver))args)))@@ -1280,20 +1342,20 @@(defmethod ->honeysql [:sql :sum-where][driver [_ arg pred]]- (->honeysql driver [:sum [:case [[pred arg]] {:default 0.0}]]))+ (->honeysql driver (mbql-clause driver :sum (mbql-clause driver :case [[pred arg]] {:default 0.0}))))(defmethod ->honeysql [:sql :count-where][driver [_ pred]]- (->honeysql driver [:sum-where 1 pred]))+ (->honeysql driver (mbql-clause driver :sum-where 1 pred)))(defmethod ->honeysql [:sql :share][driver [_ pred]]- [:/ (->honeysql driver [:count-where pred]) :%count.*])+ [:/ (->honeysql driver (mbql-clause driver :count-where pred)) :%count.*])(defmethod ->honeysql [:sql :distinct-where][driver [_ arg pred]][::h2x/distinct-count- (->honeysql driver [:case [[pred arg]]])])+ (->honeysql driver (mbql-clause driver :case [[pred arg]]))])(defmethod ->honeysql [:sql :trim][driver [_ arg]]@@ -1399,11 +1461,11 @@(defmethod ->honeysql [:sql :text][driver [_ value]]- (->honeysql driver [::cast-to-text value]))+ (->honeysql driver (mbql-clause driver ::cast-to-text value)))(defmethod ->honeysql [:sql :today][driver [_]]- (->honeysql driver [:date [:now]]))+ (->honeysql driver (mbql-clause driver :date (mbql-clause driver :now))))(mu/defmethod ->honeysql [:sql :relative-datetime] :- some?[driver [_ amount unit]]@@ -1455,8 +1517,12 @@Optional third parameter `unique-name-fn` is no longer used as of 0.42.0."([driver :- :keyword- [clause-type id-or-name opts] :- vector?]- (let [desired-alias (or (get opts driver-api/qp.add.desired-alias)+ clause :- vector?]+ (let [[clause-type id-or-name opts] (driver-api/match-one clause+ [clause-type (opts :guard :lib/uuid) id-or-name] ;; mbql5+ [clause-type id-or-name opts]+ _ clause)+ desired-alias (or (get opts driver-api/qp.add.desired-alias);; fallback behavior for anyone using SQL QP functions directly without including the stuff;; from [[metabase.query-processor.util.add-alias-info]]. We should probably disallow this;; going forward because it is liable to break@@ -1536,27 +1602,31 @@;;;; See #17536 and #18742+(defn- force-using-column-alias-opts+ [opts is-breakout]+ (cond-> (assoc opts+ driver-api/qp.add.source-alias (get opts driver-api/qp.add.desired-alias)+ driver-api/qp.add.source-table driver-api/qp.add.none+ ;; this key will tell the SQL QP not to apply casting here either.+ :qp/ignore-coercion true+ ;; used to indicate that this is a forced alias+ ::forced-alias true)+ ;; don't want to do temporal bucketing or binning inside the order by only.+ ;; That happens inside the `SELECT`+ ;; (#22831) however, we do want it in breakout+ (not is-breakout) (dissoc :temporal-unit :binning)))+(defn rewrite-fields-to-force-using-column-aliases"Rewrite `:field` clauses to force them to use the column alias regardless of where they appear."([form](rewrite-fields-to-force-using-column-aliases form {:is-breakout false}))([form {is-breakout :is-breakout}]- (driver-api/replace- form+ (driver-api/replace form+ [:field (opts :guard :lib/uuid) id-or-name] ;; mbql5+ [:field (force-using-column-alias-opts opts is-breakout) id-or-name]+[:field id-or-name opts]- [:field id-or-name (cond-> opts- true- (assoc driver-api/qp.add.source-alias (get opts driver-api/qp.add.desired-alias)- driver-api/qp.add.source-table driver-api/qp.add.none- ;; this key will tell the SQL QP not to apply casting here either.- :qp/ignore-coercion true- ;; used to indicate that this is a forced alias- ::forced-alias true)- ;; don't want to do temporal bucketing or binning inside the order by only.- ;; That happens inside the `SELECT`- ;; (#22831) however, we do want it in breakout- (not is-breakout)- (dissoc :temporal-unit :binning))])))+ [:field id-or-name (force-using-column-alias-opts opts is-breakout)])));;; +----------------------------------------------------------------------------------------------------------------+;;; | Clause Handlers |@@ -1564,11 +1634,21 @@;;; -------------------------------------------------- aggregation ---------------------------------------------------+(defmulti aggregation-name+ "Returns the name of an aggregation clause."+ {:added "0.61.0" :arglists '([driver inner-query ag-clause])}+ driver/dispatch-on-initialized-driver+ :hierarchy #'driver/hierarchy)++(defmethod aggregation-name :sql+ [_driver inner-query ag-clause]+ (driver-api/aggregation-name inner-query ag-clause))+(defmethod apply-top-level-clause [:sql :aggregation][driver _top-level-clause honeysql-form {aggregations :aggregation, :as inner-query}](let [honeysql-ags (vec (for [ag aggregations:let [ag-expr (->honeysql driver ag)- ag-name (driver-api/aggregation-name inner-query ag)+ ag-name (aggregation-name driver inner-query ag)ag-alias (->honeysql driver (h2x/identifier:field-alias(driver/escape-alias driver ag-name)))]]@@ -1623,7 +1703,9 @@[:or[:and driver-api/mbql.schema.value[:fn {:error/message "string value"} #(string? (second %))]]- driver-api/mbql.schema.FieldOrExpressionDef])+ driver-api/mbql.schema.FieldOrExpressionDef+ ;; mbql5+ [:ref :metabase.lib.schema.expression/string]])(defmulti escape-like-pattern"Handle escaping a literal string into a `LIKE` clause pattern which will match literally.@@ -1661,6 +1743,14 @@[_driver like-rhs-honeysql][:escape like-rhs-honeysql [:inline "\\"]])+(defmulti clause-value-idx+ "Returns the index of the value in a value clause."+ {:added "0.61.0" :arglists '([driver])}+ driver/dispatch-on-initialized-driver+ :hierarchy #'driver/hierarchy)++(defmethod clause-value-idx :sql [_driver] 1)+(mu/defn- generate-pattern"Generate pattern to match against in like clause. Lowercasing for case insensitive matching also happens here."[driver@@ -1669,47 +1759,49 @@post{:keys [case-sensitive] :or {case-sensitive true} :as _options}](if (= :value type)- (->> (update arg 1 #(cond-> (str pre (escape-like-pattern driver %) post)- (not case-sensitive) u/lower-case-en))+ (->> (update arg (clause-value-idx driver) #(cond-> (str pre (escape-like-pattern driver %) post)+ (not case-sensitive) u/lower-case-en))(->honeysql driver)(transform-literal-like-pattern-honeysql driver))- (let [expr (->honeysql driver (into [:concat] (remove nil?) [pre arg post]))]+ (let [expr (->honeysql driver (apply mbql-clause driver (into [:concat] (remove nil?) [pre arg post])))](if case-sensitiveexpr[:lower expr]))))(defn- uuid-field?[x]- (and (driver-api/mbql-clause? x)- (isa? (or (:effective-type (get x 2))- (let [field-id (second x)]+ (let [[opts field-id] (driver-api/match-one x+ [:field (opts :guard :lib/uuid) field-id] [opts field-id] ;; mbql5+ [:field field-id opts] [opts field-id])]+ (and (driver-api/mbql-clause? x)+ (isa? (or (:effective-type opts)(when (pos-int? field-id)(let [{:keys [base-type effective-type]}(driver-api/field (driver-api/metadata-provider) field-id)]- (or effective-type base-type))))- (:base-type (get x 2)))- :type/UUID)))+ (or effective-type base-type)))+ (:base-type opts))+ :type/UUID))))(mu/defn- maybe-cast-uuid-for-equality"For := and :!=. Comparing UUID fields against non-uuid values requires casting."[driver field arg](if (and (uuid-field? field)- ;; If the arg is a uuid we are happy especially for joins (#46558)+ ;; If the arg is a uuid we are happy especially for joins (#46558)(not (uuid-field? arg))- ;; If we could not convert the arg to a UUID then we have to cast the Field.- ;; This will not hit indexes, but then we're passing an arg that can only be compared textually.+ ;; If we could not convert the arg to a UUID then we have to cast the Field.+ ;; This will not hit indexes, but then we're passing an arg that can only be compared textually.(not (uuid? (->honeysql driver arg)))- ;; Check for inlined values+ ;; Check for inlined values(not (= (:database-type (h2x/type-info (->honeysql driver arg))) "uuid")))- [::cast-to-text field]+ (mbql-clause driver ::cast-to-text field)field))(mu/defn maybe-cast-uuid-for-text-compare"For :contains, :starts-with, and :ends-with.Comparing UUID fields against with these operations requires casting as the right side will have `%` for `LIKE` operations."- [field]+ [driver field](if (uuid-field? field)- [::cast-to-text field]+ (mbql-clause driver ::cast-to-text field)field))(defmethod ->honeysql [:sql ::cast]@@ -1721,7 +1813,7 @@;; Oracle does not support text type,;; sqlserver limits varchar to 30 in casts,;; athena cannot cast uuid to bounded varchars- (->honeysql driver [::cast expr "text"]))+ (->honeysql driver (mbql-clause driver ::cast expr "text")))(defmethod ->honeysql [:sql ::expression-literal-text-value][driver [_ value]]@@ -1737,17 +1829,17 @@(defmethod ->honeysql [:sql :starts-with][driver [_ field arg options]]- (like-clause (->honeysql driver (maybe-cast-uuid-for-text-compare field))+ (like-clause (->honeysql driver (maybe-cast-uuid-for-text-compare driver field))(generate-pattern driver nil arg "%" options) options))(defmethod ->honeysql [:sql :contains][driver [_ field arg options]]- (like-clause (->honeysql driver (maybe-cast-uuid-for-text-compare field))+ (like-clause (->honeysql driver (maybe-cast-uuid-for-text-compare driver field))(generate-pattern driver "%" arg "%" options) options))(defmethod ->honeysql [:sql :ends-with][driver [_ field arg options]]- (like-clause (->honeysql driver (maybe-cast-uuid-for-text-compare field))+ (like-clause (->honeysql driver (maybe-cast-uuid-for-text-compare driver field))(generate-pattern driver "%" arg nil options) options))(defn- parent-honeysql-col-base-type-map@@ -1816,9 +1908,17 @@[:= (->honeysql driver field-arg) nil]]honeysql-clause)))+(defn- unwrap-value-literal+ "Extract value literal from `:value` form or returns form as is if not a `:value` form."+ [maybe-value-form]+ (driver-api/match-one maybe-value-form+ [:value (opts :guard :lib/uuid) x & _] x ;; mbql5+ [:value x & _] x+ _ maybe-value-form))+(defmethod ->honeysql [:sql :!=][driver [_ field value]]- (if (nil? (driver-api/unwrap-value-literal value))+ (if (nil? (unwrap-value-literal value))[:not= (->honeysql driver (maybe-cast-uuid-for-equality driver field value)) (->honeysql driver value)](correct-null-behaviour driver [:not= (maybe-cast-uuid-for-equality driver field value) value])))@@ -1875,7 +1975,7 @@:else(->honeysql driver (driver-api/table (driver-api/metadata-provider) source-table))))-(def ^:private HoneySQLJoin+(def HoneySQLJoin"Schema for HoneySQL for a single JOIN. Used to validate that our join-handling code generates correct clauses."[:tuple;;join source and alias@@ -1892,8 +1992,7 @@(let [join-alias ((some-fn driver-api/qp.add.alias :alias) join)](assert (string? join-alias))[[(join-source driver join)- (let [table-alias (->honeysql driver (h2x/identifier :table-alias join-alias))]- [table-alias])]+ [(->honeysql driver (h2x/identifier :table-alias join-alias))]](->honeysql driver condition)]))(defn- apply-joins-honey-sql-2@@ -1972,7 +2071,7 @@{:source-table 0, :breakout 1, ...}"(into {} (map-indexed#(vector %2 %1)- [:source-table :breakout :aggregation :fields :filter :joins :order-by :page :limit])))+ [:source-table :breakout :aggregation :fields :filter :filters :joins :order-by :page :limit])))(defn- query->keys-in-application-order"Return the keys present in an MBQL `inner-query` in the order they should be processed."@@ -2068,6 +2167,18 @@(declare apply-clauses)+(defn needs-cte-for-duplicate-cols?+ "Determines if the source query has ambiguous columns names"+ [source-metadata]+ (let [source-aliases (mapv :lib/source-column-alias source-metadata)+ desired-aliases (mapv :lib/desired-column-alias source-metadata)]+ (and (seq desired-aliases)+ (> (count desired-aliases) 1)+ (apply distinct? desired-aliases)+ (every? string? desired-aliases)+ (> (count source-aliases) 1)+ (not (apply distinct? source-aliases)))))+(defn- apply-source-query"Handle a `:source-query` clause by adding a recursive `SELECT` or native query. If the source query has ambiguouscolumn names, use a `WITH` statement to rename the source columns. At the time of this writing, all source queries@@ -2084,18 +2195,10 @@:else(apply-clauses driver {} source-query))- source-aliases (mapv :lib/source-column-alias source-metadata)- desired-aliases (mapv :lib/desired-column-alias source-metadata)- duplicate-source-aliases? (and (> (count source-aliases) 1)- (not (apply distinct? source-aliases)))- needs-columns? (and (seq desired-aliases)- (> (count desired-aliases) 1)- duplicate-source-aliases?- (apply distinct? desired-aliases)- (every? string? desired-aliases))]+ desired-aliases (mapv :lib/desired-column-alias source-metadata)](mergehoneysql-form- (if needs-columns?+ (if (needs-cte-for-duplicate-cols? source-metadata);; HoneySQL cannot expand [::h2x/identifier :table "__mb_source"] in the with alias.;; This is ok since we control the alias.{:with [[[source-query-alias {:columns (mapv #(h2x/identifier :field %) desired-aliases)}]@@ -2122,7 +2225,7 @@to use this function in your own implementations of [[driver/mbql->native]], if you want to apply changes to thesame version of the query that we will ultimately be compiling.- Wants a `:lib/query` MBQL 5 query as input. Always returns an MBQL 4 **inner query**, for historical reasons."+ Wants a `:lib/query` MBQL 5 query as input. Returns an MBQL 4 **inner query**, for historical reasons, or MBQL 5 **stages**."{:changelog-test/ignore true, :arglists '([driver mbql5-query]), :added "0.42.0"}driver/dispatch-on-initialized-driver:hierarchy #'driver/hierarchy)@@ -2136,15 +2239,22 @@driver-api/->legacy-MBQL:query))-(mu/defn mbql->honeysql :- [:or :map [:tuple [:= :inline] :map]]- "Build the HoneySQL form we will compile to SQL and execute."- [driver :- :keyword- query :- :map]+(defmulti compile-mbql+ "Compiles an MBQL inner query to HoneySQL."+ {:added "0.61.0" :arglists '([driver mbql])}+ driver/dispatch-on-initialized-driver+ :hierarchy #'driver/hierarchy)++(defmethod compile-mbql :sql+ [driver query]+ (apply-clauses driver {} query))++(defn- mbql->honeysql* [driver query](if (:lib/type query)(binding [driver/*driver* driver](let [inner-query (preprocess driver query)](log/tracef "Compiling MBQL query\n%s" (u/pprint-to-str 'magenta inner-query))- (u/prog1 (apply-clauses driver {} inner-query)+ (u/prog1 (compile-mbql driver inner-query)(log/debugf "\nHoneySQL Form: %s\n%s" (u/emoji "🍯") (u/pprint-to-str 'cyan <>))(driver-api/debug> (list '🍯 <>)))))@@ -2156,6 +2266,31 @@mbql5-query (driver-api/query-from-legacy-inner-query metadata-provider database-id inner-query)](recur driver mbql5-query))))+(def ^:private experimental-mbql5-drivers {:h2 :h2-mbql5})++(defn- mbql5-experiment-report+ [driver experimental-driver query]+ (fn [{:keys [match? candidate-outcome control-outcome] :as result}]+ (when-let [report-fn @experiment/default-report-fn]+ (report-fn result))+ (when-not match?+ (log/with-context {:experimental-mbql5-driver :mismatch+ :driver driver+ :experimental-driver experimental-driver}+ (log/warnf "MBQL5 experiment mismatch:\nQuery: %s\nControl result: %s\nCandidate result: %s"+ query control-outcome candidate-outcome)))))++(mu/defn mbql->honeysql :- [:or :map [:tuple [:= :inline] :map]]+ "Build the HoneySQL form we will compile to SQL and execute."+ [driver :- :keyword+ query :- :map]+ (if-let [experimental-driver (experimental-mbql5-drivers driver)]+ (experiment/experiment {:name :experimental-mbql5-driver+ :report-fn (mbql5-experiment-report driver experimental-driver query)}+ (mbql->honeysql* driver query)+ (mbql->honeysql* experimental-driver query))+ (mbql->honeysql* driver query)))+;;;; MBQL -> Native(mu/defn mbql->native :- [:mapdiff --git a/src/metabase/driver/sql_jdbc/actions.clj b/src/metabase/driver/sql_jdbc/actions.cljindex f8981c1e166..219bcd17baa 100644--- a/src/metabase/driver/sql_jdbc/actions.clj+++ b/src/metabase/driver/sql_jdbc/actions.clj@@ -124,7 +124,7 @@(if-let [sql-type (type->sql-type base-type)](h2x/cast sql-type value)(try- (sql.qp/->honeysql driver [:value value field])+ (sql.qp/->honeysql driver (sql.qp/mbql-clause driver :value value field))(catch Exception e(throw (ex-info (str "column cast failed: " (pr-str col-name)){:column col-namediff --git a/src/metabase/driver/sql_mbql5.clj b/src/metabase/driver/sql_mbql5.cljnew file mode 100644index 00000000000..85dfc0b928e--- /dev/null+++ b/src/metabase/driver/sql_mbql5.clj@@ -0,0 +1,199 @@+(ns metabase.driver.sql-mbql5+ "An abstract driver to allow for opt-in MBQL5 compilation."+ (:refer-clojure :exclude [mapv get-in])+ (:require+ [honey.sql.helpers :as sql.helpers]+ [medley.core :as m]+ [metabase.driver :as driver]+ [metabase.driver-api.core :as driver-api]+ [metabase.driver.sql.parameters.substitution :as sql.params.substitution]+ [metabase.driver.sql.query-processor :as sql.qp]+ [metabase.lib.core :as lib]+ [metabase.lib.options :as lib.options]+ [metabase.lib.schema.common :as lib.schema.common]+ [metabase.lib.util :as lib.util]+ [metabase.query-processor.parameters.dates :as qp.params.dates]+ [metabase.query-processor.parameters.operators :as qp.params.ops]+ [metabase.query-processor.util.add-alias-info :as add]+ [metabase.util.honey-sql-2 :as h2x]+ [metabase.util.malli :as mu]+ [metabase.util.performance :refer [mapv get-in]]))++(driver/register! :sql-mbql5, :parent :sql, :abstract? true)++(defmethod sql.qp/preprocess :sql-mbql5+ [_driver mbql5-query]+ (-> mbql5-query+ driver-api/nest-breakouts-in-stages-with-window-aggregation+ driver-api/nest-expressions+ driver-api/add-alias-info+ :stages))++(defn- stage->honeysql [driver stage]+ (cond+ (:persisted-info/native stage)+ (sql.qp/sql-source-query (:persisted-info/native stage) nil)++ (= (:lib/type stage) :mbql.stage/native)+ (sql.qp/sql-source-query (:native stage) (:params stage))++ :else+ (binding [sql.qp/*inner-query* stage]+ (#'sql.qp/apply-top-level-clauses driver {} stage))))++(defn- stages->honeysql [driver stages]+ (first+ (reduce+ (fn [[prev-hsql prev-stage] stage]+ (let [stage-hsql (stage->honeysql driver stage)]+ (if prev-hsql+ (let [table-alias (sql.qp/->honeysql driver (h2x/identifier :table-alias sql.qp/source-query-alias))+ columns-metadata (get-in prev-stage [:lib/stage-metadata :columns])+ desired-aliases (mapv :lib/desired-column-alias columns-metadata)+ cur-hsql (cond-> (assoc stage-hsql :from [[prev-hsql [table-alias]]])+ (sql.qp/needs-cte-for-duplicate-cols? columns-metadata)+ (assoc :with [[[sql.qp/source-query-alias {:columns (mapv #(h2x/identifier :field %) desired-aliases)}]+ prev-hsql]]+ :from [[table-alias]])++ ;; Clear the default select if it's just :* so add-default-select can recompute it+ (= (:select stage-hsql) [[:*]])+ (dissoc :select))]+ [(#'sql.qp/add-default-select driver cur-hsql) stage])+ [stage-hsql stage])))+ [nil nil]+ stages)))++(defmethod sql.qp/compile-mbql :sql-mbql5+ [driver stages]+ (stages->honeysql driver stages))++(defn- predicates->honeysql [driver predicates]+ (let [predicates (mapv #(sql.qp/->honeysql driver %) predicates)]+ (if (= 1 (count predicates))+ (first predicates)+ (into [:and] predicates))))++(defmethod sql.qp/apply-top-level-clause [:sql-mbql5 :filters]+ [driver _ honeysql-form {:keys [filters]}]+ (sql.helpers/where honeysql-form (predicates->honeysql driver filters)))++(defmethod sql.qp/join-source :sql-mbql5+ [driver {:keys [stages]}]+ (stages->honeysql driver stages))++(mu/defmethod sql.qp/join->honeysql :sql-mbql5 :- sql.qp/HoneySQLJoin+ [driver {:keys [conditions] :as join} :- [:ref :metabase.lib.schema.join/join]]+ (let [join-alias ((some-fn driver-api/qp.add.alias :alias) join)]+ (assert (string? join-alias))+ [[(sql.qp/join-source driver join)+ [(sql.qp/->honeysql driver (h2x/identifier :table-alias join-alias))]]+ (predicates->honeysql driver conditions)]))++(defn- agg-by-id [aggs agg-uuid]+ (m/find-first #(= (lib.options/uuid %) agg-uuid) aggs))++;; Should mirror the logic in `sql.qp/->honeysql [:sql :aggregation]`+(defmethod sql.qp/->honeysql [:sql-mbql5 :aggregation]+ [driver [_ _opts _agg-uuid :as clause]]+ (driver-api/match-one clause+ [:aggregation {driver-api/qp.add.desired-alias desired-alias} _agg-uuid]+ (sql.qp/->honeysql driver (h2x/identifier :field-alias desired-alias))++ [:aggregation {driver-api/qp.add.source-alias source-alias} _agg-uuid]+ (sql.qp/->honeysql driver (h2x/identifier :field-alias source-alias))++ [:aggregation {:lib/source-name source-name} _agg-uuid]+ (sql.qp/->honeysql driver (h2x/identifier :field-alias source-name))++ [:aggregation _opts agg-uuid]+ (let [agg (agg-by-id (:aggregation sql.qp/*inner-query*) agg-uuid)]+ (&recur agg))++ [:distinct & _]+ (sql.qp/->honeysql driver (h2x/identifier :field-alias :count))++ [#{:+ :- :* :/} & _]+ (sql.qp/->honeysql driver &match)++ [:offset {:lib/source-name source-name} _expr _n]+ (sql.qp/->honeysql driver (h2x/identifier :field-alias source-name))++ [ag-type & _]+ (sql.qp/->honeysql driver (h2x/identifier :field-alias ag-type))))++(defmethod sql.qp/over-order-by->honeysql :sql-mbql5+ [driver aggregations [direction _opts expr]]+ (if (lib.util/clause-of-type? expr :aggregation)+ (let [[_op _opts agg-uuid] expr+ aggregation (agg-by-id aggregations agg-uuid)]+ (when-not (#'sql.qp/contains-clause? #{:cum-count :cum-sum :offset} aggregation)+ [(sql.qp/->honeysql driver aggregation) direction]))+ [(sql.qp/->honeysql driver expr) direction]))++(defmethod sql.qp/->honeysql [:sql-mbql5 :value]+ [driver [op {:keys [base-type effective-type]} value]]+ ;; We need to rename just these keys for `sql.qp/->honeysql [:sql :value]`+ ((get-method sql.qp/->honeysql [:sql op]) driver [op value {:base_type base-type :effective_type effective-type}]))++(defmethod sql.qp/->honeysql [:sql-mbql5 :case]+ [driver [op _opts cases opts-or-default]]+ ;; Handle the default options coming in as a map from `mbql-clause`, or as a single value from ->mbql5+ ((get-method sql.qp/->honeysql [:sql op]) driver [op cases (if (map? opts-or-default)+ opts-or-default+ {:default opts-or-default})]))++;; For clauses that DO NOT have their opts propagated+(doseq [op [:!= :* :+ :- :/ :< :<= := :> :>= :abs :absolute-datetime :aggregation-options :and :asc :avg :between+ ::sql.qp/cast ::sql.qp/cast-to-text :ceil :coalesce :concat :count :count-where :cum-count :cum-sum :date+ :datetime-add :datetime-diff :datetime-subtract :desc :distinct :distinct-where :exp+ ::sql.qp/expression-literal-text-value :float :floor :integer :length :log :lower :ltrim :max :median :min+ ::sql.qp/nfc-path :not :now :or :percentile :power :relative-datetime :replace :round :rtrim :share :sqrt+ :stddev :substring :sum :sum-where :temporal-extract :text :time :today :trim :upper :var]]+ (defmethod sql.qp/->honeysql [:sql-mbql5 op]+ [driver [op _opts & args]]+ ((get-method sql.qp/->honeysql [:sql op]) driver (into [op] args))))++;; For clauses that DO have their opts propagated+(doseq [op [:expression :field :datetime :contains :starts-with :ends-with]]+ (defmethod sql.qp/->honeysql [:sql-mbql5 op]+ [driver [op opts & args]]+ ((get-method sql.qp/->honeysql [:sql op]) driver (cond-> (into [op] args) opts (conj opts)))))++(defmethod sql.qp/mbql-clause-with-opts :sql-mbql5+ [_driver tag opts & args]+ (into [tag (lib.schema.common/normalize-options-map opts)] args))++(defmethod sql.qp/expression-by-name :sql-mbql5+ [_driver inner-query expression-name]+ (m/find-first #(= expression-name (lib.util/expression-name %)) (:expressions inner-query)))++(defmethod sql.qp/aggregation-name :sql-mbql5+ [_driver inner-query ag-clause]+ (or (::add/desired-alias (lib/options ag-clause))+ (:name (lib/options ag-clause))+ (lib/column-name inner-query ag-clause)))++(defmethod sql.qp/clause-value-idx :sql-mbql5 [_driver] 2)++(defmethod sql.qp/breakout-options-index :sql-mbql5 [_driver] 1)++(mu/defmethod sql.params.substitution/field->clause :sql-mbql5 :- :mbql.clause/field+ [driver field other-opts]+ (sql.qp/mbql-clause-with-opts driver :field+ (merge {:base-type (:base-type field)+ driver-api/qp.add.source-table (:table-id field)+ ::sql.params.substitution/compiling-field-filter? true}+ other-opts)+ (:id field)))++(defmethod sql.params.substitution/to-clause :sql-mbql5 [_driver param] (qp.params.ops/to-clause param))++(defmethod sql.params.substitution/desugar-filter-clause :sql-mbql5 [_driver filter-clause] (lib/desugar-filter-clause filter-clause))++(defmethod sql.params.substitution/wrap-value-literals-in-mbql :sql-mbql5+ [_driver mbql]+ #_{:clj-kondo/ignore [:deprecated-var]}+ (driver-api/wrap-value-literals-in-mbql5 mbql))++(defmethod sql.params.substitution/date-string->filter :sql-mbql5 [_driver date-string field] (qp.params.dates/date-string->filter date-string field))diff --git a/src/metabase/driver_api/core.clj b/src/metabase/driver_api/core.cljindex a736e4e2987..2e6e4b4adf3 100644--- a/src/metabase/driver_api/core.clj+++ b/src/metabase/driver_api/core.clj@@ -167,8 +167,8 @@qp.util.transformations.nest-breakouts/nest-breakouts-in-stages-with-window-aggregationqp.util/default-query->remarkqp.util/query->remark- qp.wrap-value-literals/unwrap-value-literalqp.wrap-value-literals/wrap-value-literals-in-mbql+ qp.wrap-value-literals/wrap-value-literals-in-mbql5qp.writeback/execute-write-sql!qp/process-querysecrets/clean-secret-properties-from-detailsdiff --git a/src/metabase/lib/convert.cljc b/src/metabase/lib/convert.cljcindex 0460c558d24..c11a1eac55a 100644--- a/src/metabase/lib/convert.cljc+++ b/src/metabase/lib/convert.cljc@@ -445,7 +445,7 @@(defmethod ->mbql5 ::string-comparison[[tag opts & args :as clause]]- (if (> (count args) 2)+ (if (or (> (count args) 2) (map? opts));; Multi-arg, MBQL 5 style: [tag {opts...} x y z ...](lib.options/ensure-uuid (into [tag opts] (map ->mbql5 args)));; Two-arg, legacy style: [tag x y] or [tag x y opts].diff --git a/src/metabase/lib/schema/mbql_clause.cljc b/src/metabase/lib/schema/mbql_clause.cljcindex c78a5472050..d48d53c8bba 100644--- a/src/metabase/lib/schema/mbql_clause.cljc+++ b/src/metabase/lib/schema/mbql_clause.cljc@@ -96,7 +96,7 @@(when (sequential? x)(if ((some-fn map? nil?) (second x))x- (into [(first x) {}] (rest x)))))+ (into [(first x) {:lib/uuid (str (random-uuid))}] (rest x)))));;; TODO: Support options more nicely - these don't allow for overriding the options, but we have a few cases where that;;; is necessary. See for example the inclusion of `string-filter-options` in [[metabase.lib.filter]].diff --git a/src/metabase/query_processor/middleware/wrap_value_literals.clj b/src/metabase/query_processor/middleware/wrap_value_literals.cljindex 3f1af6ba93d..8da92e2626e 100644--- a/src/metabase/query_processor/middleware/wrap_value_literals.clj+++ b/src/metabase/query_processor/middleware/wrap_value_literals.clj@@ -277,15 +277,6 @@;;; Tangentially-related nonsense not used by the middleware;;;-;;; TODO (Cam 8/22/25) FIXME: This is used in exactly one place: the SQL QP... so why does it live in a QP middleware-;;; namespace? Nobody knows.-(defn unwrap-value-literal- "Extract value literal from `:value` form or returns form as is if not a `:value` form."- [maybe-value-form]- (match/match-one maybe-value-form- [:value x & _] x- _ maybe-value-form))-(defn- type-info-no-query"This is like [[type-info*]] but specifically for supporting the legacy/deprecated [[wrap-value-literals-in-mbql]]function."@@ -303,7 +294,7 @@{:base-type expr-type:effective-type expr-type})))-(mu/defn wrap-value-literals-in-mbql :- [:cat :keyword [:* :any]]+(mu/defn wrap-value-literals-in-mbql5 :- [:cat :keyword [:* :any]]"Given a normalized legacy MBQL query (important to desugar forms like `[:does-not-contain ...]` -> `[:not [:contains...]]`), walks over the clause and annotates literals with type information.@@ -329,5 +320,12 @@[$mbql](fn [clause](wrap-value-literals-in-clause nil nil clause)))- first)))- lib/->legacy-MBQL))+ first)))))++(mu/defn wrap-value-literals-in-mbql :- [:cat :keyword [:* :any]]+ "Wrapper around `wrap-value-literals-in-mbql5` that converts the clause back to legacy MBQL.+ DEPRECATED: This is for legacy compatibility and should not be used in new code."+ {:deprecated "0.57.0"}+ [mbql :- [:cat :keyword [:* :any]]]+ #_{:clj-kondo/ignore [:deprecated-var]}+ (lib/->legacy-MBQL (wrap-value-literals-in-mbql5 mbql)))diff --git a/src/metabase/util/experiment.cljc b/src/metabase/util/experiment.cljcindex 0f239704485..1f58f4c6c2a 100644--- a/src/metabase/util/experiment.cljc+++ b/src/metabase/util/experiment.cljc@@ -35,9 +35,8 @@[f](reset! experiments-enabled-fn f))-(defonce ^{:private true- :doc "Atom holding the default report function. Set at application startup via [[set-default-report-fn!]].- Nil means no reporting. Can be overridden per-experiment via :report-fn in the opts map."}+(defonce ^{:doc "Atom holding the default report function. Set at application startup via [[set-default-report-fn!]].+ Nil means no reporting. Can be overridden per-experiment via :report-fn in the opts map."}default-report-fn (atom nil))(defn set-default-report-fn!diff --git a/src/metabase/util/honey_sql_2.clj b/src/metabase/util/honey_sql_2.cljindex 48c5aee1239..461f2b2a9b3 100644--- a/src/metabase/util/honey_sql_2.clj+++ b/src/metabase/util/honey_sql_2.clj@@ -436,7 +436,7 @@"HoneySQL form that should be used to get the current `datetime` (or equivalent), e.g. `:%now`."[db-type](case db-type- :h2 (with-database-type-info :%now "timestamp")+ (:h2 :h2-mbql5) (with-database-type-info :%now "timestamp"):mysql (with-database-type-info [:now [:inline 6]] "timestamp"):postgres (with-database-type-info :%now "timestamptz")))@@ -525,6 +525,10 @@:else(dateadd-h2 unit amount hsql-form)))+(defmethod add-interval-honeysql-form :h2-mbql5+ [db-type hsql-form amount unit]+ ((get-method add-interval-honeysql-form :h2) db-type hsql-form amount unit))+(defmethod add-interval-honeysql-form :default[db-type hsql-form amount unit](throw (ex-info (clojure.core/format (str "metabase.util.honey-sql-2/add-interval-honeysql-form not implemented for db-type %s. "diff --git a/test/metabase/driver/h2_test.clj b/test/metabase/driver/h2_test.cljindex 0bb540db63f..d5e4d18b86c 100644--- a/test/metabase/driver/h2_test.clj+++ b/test/metabase/driver/h2_test.clj@@ -21,11 +21,34 @@[metabase.query-processor.preprocess :as qp.preprocess][metabase.query-processor.test :as qp][metabase.test :as mt]+ [metabase.test.data.datasets :as mtd]+ [metabase.test.data.env :as tx.env]+ [metabase.test.data.interface :as tx][metabase.util.honey-sql-2 :as h2x][toucan2.core :as t2]))(set! *warn-on-reflection* true)+;; temporary hack to run all tests against both the old :h2 and new mbql5 :h2+;; remove this once :h2 does mbql5 by default!+(defn- test-driver [driver thunk]+ (when (contains? (tx.env/test-drivers) driver)+ (testing (str "\n" driver "\n")+ (driver/with-driver (tx/the-driver-with-test-extensions driver)+ (thunk))+ ;; the above is the original definition of test-driver, but we add in+ ;; this clause to avoid having to rewrite all the tests below twice:+ (when (= driver :h2)+ (driver/with-driver (tx/the-driver-with-test-extensions :h2-mbql5)+ (thunk))))))++(use-fixtures :each (fn [f]+ ;; NB: because of test parallelism, this *will* affect other non-h2+ ;; tests, but the check above in the test-driver function will+ ;; prevent it from actually doing anything different in those tests.+ (with-redefs [mtd/-test-driver test-driver]+ (f))))+(deftest ^:parallel parse-connection-string-test(testing "Check that the functions for exploding a connection string's options work as expected"(is (= ["file:my-file" {"OPTION_1" "TRUE", "OPTION_2" "100", "LOOK_I_INCLUDED_AN_EXTRA_SEMICOLON" "NICE_TRY"}]diff --git a/test/metabase/driver/sql/query_processor_test.clj b/test/metabase/driver/sql/query_processor_test.cljindex db9b7ffe998..cb7359185b0 100644--- a/test/metabase/driver/sql/query_processor_test.clj+++ b/test/metabase/driver/sql/query_processor_test.clj@@ -9,6 +9,7 @@[metabase.driver.sql.query-processor :as sql.qp][metabase.driver.sql.query-processor-test-util :as sql.qp-test-util][metabase.driver.sql.query-processor.deprecated]+ [metabase.lib.convert :as lib.convert][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.lib.options :as lib.options]@@ -17,6 +18,7 @@[metabase.lib.test-util.macros :as lib.tu.macros][metabase.lib.test-util.notebook-helpers :as lib.tu.notebook][metabase.lib.test-util.places-cam-likes-metadata-provider :as lib.tu.places-cam-likes-metadata-provider]+ [metabase.lib.util :as lib.util][metabase.query-processor.compile :as qp.compile][metabase.query-processor.middleware.limit :as limit][metabase.query-processor.preprocess :as qp.preprocess]@@ -221,12 +223,18 @@(testing "params from source queries should get passed in to the top-level. Semicolons should be removed"(is (= {:query "SELECT \"__mb_source\".* FROM (SELECT * FROM some_table WHERE name = ?) AS \"__mb_source\" WHERE (\"__mb_source\".\"name\" <> ?) OR (\"__mb_source\".\"name\" IS NULL)":params ["Cam" "Lucky Pigeon"]}- (sql.qp/mbql->native- :h2- (lib.tu.macros/mbql-query venues- {:source-query {:native "SELECT * FROM some_table WHERE name = ?;", :params ["Cam"]}- :source-metadata [{:name "name", :display_name "Name", :base_type :type/Integer}]- :filter [:!= *name/Integer "Lucky Pigeon"]}))))))))+ (->> {:source-query {:native "SELECT * FROM some_table WHERE name = ?;", :params ["Cam"]}+ :source-metadata [{:name "name", :display_name "Name", :base_type :type/Integer}]+ :filter [:!= *name/Integer "Lucky Pigeon"]}+ (lib.tu.macros/mbql-query venues)+ (lib.convert/->mbql5)+ (lib/query meta/metadata-provider)+ (sql.qp/mbql->native :h2))))))))++(defn- legacy-join->honeysql [driver join]+ (sql.qp/join->honeysql driver (if (isa? driver/hierarchy driver :sql-mbql5)+ (lib.convert/->mbql5 (#'lib.util/join->pipeline join))+ join)))(deftest ^:parallel joins-against-native-queries-test(testing "Joins against native SQL queries should get converted appropriately! make sure correct HoneySQL is generated"@@ -237,33 +245,31 @@[:=(h2x/with-database-type-info (h2x/identifier :field "PUBLIC" "CHECKINS" "VENUE_ID") "integer")(h2x/identifier :field "card" "id")]]- (sql.qp/join->honeysql :h2- (lib.tu.macros/$ids checkins+ (legacy-join->honeysql :h2+ {:source-query {:native "SELECT * FROM VENUES;", :params []}+ :alias "card"+ :strategy :left-join+ :condition [:=+ [:field (meta/id :checkins :venue-id) {::add/source-table (meta/id :checkins)+ ::add/source-alias "VENUE_ID"}]+ [:field "id" {:join-alias "card"+ :base-type :type/Integer+ ::add/source-table "card"+ ::add/source-alias "id"}]]})))))))++(defn- compile-join [driver]+ (driver/with-driver driver+ (qp.store/with-metadata-provider meta/metadata-provider+ (let [join (legacy-join->honeysql driver{:source-query {:native "SELECT * FROM VENUES;", :params []}:alias "card":strategy :left-join:condition [:=- [:field %venue-id {::add/source-table $$checkins- ::add/source-alias "VENUE_ID"}]- [:field "id" {:join-alias "card"- :base-type :type/Integer+ [:field (meta/id :checkins :id) {::add/source-table (meta/id :checkins)+ ::add/source-alias "VENUE_ID"}]+ [:field "id" {:base-type :type/Text::add/source-table "card"- ::add/source-alias "id"}]]}))))))))--(defn- compile-join [driver]- (driver/with-driver driver- (qp.store/with-metadata-provider meta/metadata-provider- (let [join (sql.qp/join->honeysql- driver- {:source-query {:native "SELECT * FROM VENUES;", :params []}- :alias "card"- :strategy :left-join- :condition [:=- [:field (meta/id :checkins :id) {::add/source-table (meta/id :checkins)- ::add/source-alias "VENUE_ID"}]- [:field "id" {:base-type :type/Text- ::add/source-table "card"- ::add/source-alias "id"}]]})]+ ::add/source-alias "id"}]]})](sql.qp/format-honeysql driver {:join join})))));;; Ok to hardcode driver names here because it's for general HoneySQL compilation behavior and not something that needsdiff --git a/test/metabase/driver/sql_jdbc_test.clj b/test/metabase/driver/sql_jdbc_test.cljindex 93852bdc357..555405658d7 100644--- a/test/metabase/driver/sql_jdbc_test.clj+++ b/test/metabase/driver/sql_jdbc_test.clj@@ -233,48 +233,52 @@[[uuid]] (lib/ends-with col (str uuid))[[uuid]] (lib/contains col (str uuid))- ;; Test partial uuid values+ ;; Test partial uuid values[[uuid]] (lib/contains col (subs (str uuid) 0 1))[[uuid]] (lib/starts-with col (subs (str uuid) 0 1))[[uuid]] (lib/ends-with col (subs (str uuid) (dec (count (str uuid)))))- ;; Cannot match a uuid, but should not blow up+ ;; Cannot match a uuid, but should not blow up[[uuid]] (lib/!= col "q")[] (lib/= col "q")[] (lib/starts-with col "q")[] (lib/ends-with col "q")[] (lib/contains col "q")- ;; empty/null handling+ ;; empty/null handling[] (lib/is-empty col)[[uuid]] (lib/not-empty col)[] (lib/is-null col)[[uuid]] (lib/not-null col)- ;; nil value handling+ ;; nil value handling[[uuid]] (lib/!= col nil)[] (lib/= col nil))- (testing ":= uses indexable query"- (is (=? [:= [:metabase.util.honey-sql-2/identifier :field [(second (lib/->legacy-MBQL (lib/ref col)))]]- (some-fn #(= uuid %)- #(= [:metabase.util.honey-sql-2/typed- [:cast (str uuid) [:raw "uuid"]]- {:database-type "uuid"}]- %))]- (sql.qp/->honeysql- driver/*driver*- [:=- (lib/->legacy-MBQL (lib/ref col))- [:value (str uuid) {:base_type :type/UUID}]])))- (is (=? [:= [:metabase.util.honey-sql-2/identifier :field [(second (lib/->legacy-MBQL (lib/ref col)))]]- (some-fn #(= uuid %)- #(= [:metabase.util.honey-sql-2/typed- [:cast (str uuid) [:raw "uuid"]]- {:database-type "uuid"}]- %))]- (sql.qp/->honeysql- driver/*driver*- [:= (lib/->legacy-MBQL (lib/ref col)) uuid])))))))+ (let [mbql5-driver? (isa? driver/hierarchy driver/*driver* :sql-mbql5)+ field (get (lib/ref col) 2)+ col-ref (cond-> (lib/ref col)+ (not mbql5-driver?) lib/->legacy-MBQL)]+ (testing ":= uses indexable query"+ (is (=? [:= [:metabase.util.honey-sql-2/identifier :field [field]]+ (some-fn #(= uuid %)+ #(= [:metabase.util.honey-sql-2/typed+ [:cast (str uuid) [:raw "uuid"]]+ {:database-type "uuid"}]+ %))]+ (sql.qp/->honeysql+ driver/*driver*+ (sql.qp/mbql-clause driver/*driver*+ := col-ref+ (sql.qp/mbql-clause-with-opts driver/*driver* :value {:base_type :type/UUID} (str uuid))))))+ (is (=? [:= [:metabase.util.honey-sql-2/identifier :field [field]]+ (some-fn #(= uuid %)+ #(= [:metabase.util.honey-sql-2/typed+ [:cast (str uuid) [:raw "uuid"]]+ {:database-type "uuid"}]+ %))]+ (sql.qp/->honeysql+ driver/*driver*+ (sql.qp/mbql-clause driver/*driver* := col-ref uuid)))))))))(deftest query-canceled-test?(testing "walks a chain of exceptions"diff --git a/test/metabase/query_processor/date_bucketing_test.clj b/test/metabase/query_processor/date_bucketing_test.cljindex 762c404fe98..2503d7f456c 100644--- a/test/metabase/query_processor/date_bucketing_test.clj+++ b/test/metabase/query_processor/date_bucketing_test.clj@@ -1810,7 +1810,7 @@#t "2022-03-31T00:00:00"#t "2022-03-31T00:00:00-00:00"]](testing (format "%d %s ^%s %s" n unit (.getCanonicalName (class t)) (pr-str t))- (let [march-31 (sql.qp/->honeysql driver/*driver* [:absolute-datetime t :day])+ (let [march-31 (sql.qp/->honeysql driver/*driver* (sql.qp/mbql-clause driver/*driver* :absolute-datetime t :day))june-31 (sql.qp/add-interval-honeysql-form driver/*driver* march-31 n unit)checkins (mt/with-metadata-provider (mt/id)(sql.qp/->honeysql driver/*driver* (lib.metadata/table (qp.store/metadata-provider) (mt/id :checkins))))diff --git a/test/metabase/query_processor/expression_aggregations_test.clj b/test/metabase/query_processor/expression_aggregations_test.cljindex 48d2abdbeee..41265a92615 100644--- a/test/metabase/query_processor/expression_aggregations_test.clj+++ b/test/metabase/query_processor/expression_aggregations_test.clj@@ -4,6 +4,7 @@[clojure.test :refer :all][metabase.driver :as driver][metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata][metabase.lib.test-util :as lib.tu][metabase.query-processor.test :as qp][metabase.query-processor.test-util :as qp.test-util]@@ -395,3 +396,26 @@mt/rows(map second)(map sort))))))))++(deftest ^:parallel aggregation-function-inside-expression-in-aggregation-test+ (testing "should allow using aggregation functions inside expressions in aggregation (metabase#52611)"+ (mt/test-drivers (mt/normal-driver-select {:+features [:expression-aggregations :binning]})+ (let [mp (mt/metadata-provider)+ orders (lib.metadata/table mp (mt/id :orders))+ total (lib.metadata/field mp (mt/id :orders :total))+ subtotal (lib.metadata/field mp (mt/id :orders :subtotal))+ query (-> (lib/query mp orders)+ (lib/aggregate+ (lib/case [[(lib/> (lib/sum total) 10) (lib/sum total)]]+ (lib/sum subtotal)))+ (lib/breakout (lib/with-binning total {:strategy :bin-width :bin-width 20})))]+ (is (= [[-60 26.96]+ [0 851.99]+ [20 71843.32]+ [40 204530.21]+ [60 283033.59]+ [80 233639.15]+ [100 270513.03]+ [120 317763.17]+ [140 128488.72]]+ (mt/formatted-rows [int 2.0] (qp/process-query query))))))))diff --git a/test/metabase/test/data/h2_mbql5.clj b/test/metabase/test/data/h2_mbql5.cljnew file mode 100644index 00000000000..e43e78486b4--- /dev/null+++ b/test/metabase/test/data/h2_mbql5.clj@@ -0,0 +1,2 @@+(ns metabase.test.data.h2-mbql5+ "Placeholder; delete me once experimental-mbql5-driver goes away.")