Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Enable MBQL5 compilation for drivers (#71439)

Riley Thompson3728dbb5a510
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 291a30b91c5..c85b02c869a 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -1061,6 +1061,7 @@
metabase.lib.init
metabase.lib.metadata
metabase.lib.metadata.protocols
+ metabase.lib.options
metabase.lib.schema
metabase.lib.schema.actions
metabase.lib.schema.aggregation
@@ -1757,6 +1758,7 @@
metabase.query-processor.streaming.interface
metabase.query-processor.timezone
metabase.query-processor.util
+ metabase.query-processor.util.add-alias-info
metabase.query-processor.util.persisted-cache}
:uses #{analytics
analytics-interface
diff --git a/docs/developers-guide/driver-changelog.md b/docs/developers-guide/driver-changelog.md
index 7f8f7044c22..6e8b576b409 100644
--- a/docs/developers-guide/driver-changelog.md
+++ b/docs/developers-guide/driver-changelog.md
@@ -6,6 +6,29 @@ title: Driver interface changelog
## Metabase 0.61.0
+- Added the following driver multimethods to support MBQL5 compilation migration:
+ - `sql.qp/compile-mbql` - Compiles an MBQL inner query to HoneySQL.
+ - `sql.qp/mbql-clause-with-opts` - Returns an MBQL clause in the desired MBQL format of the driver.
+ - `sql.qp/expression-by-name` - Gets an expression from a query or stage (`*inner-query`) by name.
+ - `sql.qp/aggregation-name` - Returns the name of an aggregation clause.
+ - `sql.qp/over-order-by->honeysql` - Returns the HoneySQL for an order by clause in the over clause of a window function.
+ - `sql.qp/clause-value-idx` - Returns the index of the value in a value clause.
+ - `sql.qp/breakout-options-index` - Returns the index of options in a breakout clause.
+ - `sql.params.substitution/field->clause` - Returns an MBQL field clause with the given options.
+ - `sql.params.substitution/to-clause` - Helper to dispatch to `params.ops/to-clause` or `qp.params.ops/to-clause`.
+ - `sql.params.substitution/desugar-filter-clause` - Helper to dispatch to `driver-api/desugar-filter-clause` or `lib/desugar-filter-clause`.
+ - `sql.params.substitution/wrap-value-literals-in-mbql` - Helper to dispatch to `driver-api/wrap-value-literals-in-mbql` or `driver-api/wrap-value-literals-in-mbql5`.
+ - `sql.params.substitution/date-string->filter` - Helper to dispatch to `params.dates/date-string->filter` or `qp.params.dates/date-string->filter`.
+
+ These methods have implementations for the `:sql` and `:sql-mbql5` drivers. Concrete drivers should *not* need to
+ implement these methods. Drivers can opt-in to MBQL5 compilation by adding the `:sql-mbql5` driver as a parent, and updating the `sql.qp/->honeysql`
+ methods to handle the clause options argument as the second parameter. See the `:h2` driver in https://github.com/metabase/metabase/pull/71439 for
+ an example. Drivers will need to be migrated to work with MBQL5 compilation over the next three releases by v64. After v64 these methods will be deprecated
+ in favour of the `:sql-mbql5` implementations once all drivers have been migrated.
+
+- Added a `driver` parameter to `sql.qp/maybe-cast-uuid-for-text-compare`. Any drivers that call this function should
+ update it to pass in the `driver` parameter now. An example is in the Snowflake driver's `string-filter` function.
+
- `driver/field-reference-mlv2`, deprecated in 0.57.0, has now been removed.
- `metabase.driver.sql/set-role-statement` has been deprecated in favor of
diff --git a/enterprise/backend/test/metabase_enterprise/impersonation/driver_test.clj b/enterprise/backend/test/metabase_enterprise/impersonation/driver_test.clj
index 2bd1dc6ee5a..c3610d74377 100644
--- a/enterprise/backend/test/metabase_enterprise/impersonation/driver_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/impersonation/driver_test.clj
@@ -710,7 +710,7 @@
(is (thrown-with-msg?
clojure.lang.ExceptionInfo
;; I've seen different error messages here, not 100% sure why but the important thing is that this fails
- #"(?s)SQL compilation error.*(?:(?:operation cannot be performed)|(?:Object.*does not exist or not authorized))"
+ #"(?s)(?:SQL compilation error.*(?:(?:operation cannot be performed)|(?:Object.*does not exist or not authorized))|Cannot perform SELECT)"
(mt/run-mbql-query venues
{:aggregation [[:count]]})))
diff --git a/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj b/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj
index 6f246fb1729..e2b7c3ced12 100644
--- a/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj
@@ -55,9 +55,12 @@
(qp.store/with-metadata-provider (mt/id)
(sql.qp/->honeysql
(or driver/*driver* :h2)
- [:field field-id {::add/source-table (mt/id table-key)
- ::add/source-alias field-name
- ::add/desired-alias field-name}])))))
+ (sql.qp/mbql-clause-with-opts driver/*driver*
+ :field
+ {::add/source-table (mt/id table-key)
+ ::add/source-alias field-name
+ ::add/desired-alias field-name}
+ field-id))))))
(defn- venues-category-mbql-gtap-def []
{:query (mt/mbql-query venues)
diff --git a/modules/drivers/snowflake/src/metabase/driver/snowflake.clj b/modules/drivers/snowflake/src/metabase/driver/snowflake.clj
index ae9db9546c9..dbb2af621ea 100644
--- a/modules/drivers/snowflake/src/metabase/driver/snowflake.clj
+++ b/modules/drivers/snowflake/src/metabase/driver/snowflake.clj
@@ -1038,7 +1038,7 @@
(defn- string-filter
[driver str-filter field arg {:keys [case-sensitive] :or {case-sensitive true} :as options}]
- (let [casted-field (sql.qp/->honeysql driver (sql.qp/maybe-cast-uuid-for-text-compare field))]
+ (let [casted-field (sql.qp/->honeysql driver (sql.qp/maybe-cast-uuid-for-text-compare driver field))]
[str-filter
(if case-sensitive casted-field [:lower casted-field])
(get-string-filter-arg driver arg options)]))
diff --git a/src/metabase/driver/h2.clj b/src/metabase/driver/h2.clj
index 77a6d8ee07a..d290d6466d1 100644
--- a/src/metabase/driver/h2.clj
+++ b/src/metabase/driver/h2.clj
@@ -44,6 +44,12 @@
(driver/register! :h2, :parent #{:sql-jdbc ::like-escape-char-built-in/like-escape-char-built-in})
+;; h2 can be used to generate mbql5 natively, but this is not the default yet.
+;; we need to gather more data from the experiment (see query-processor/mbql->honeysql)
+;; before we can switch this over. once that happens, make regular :h2 have
+;; :sql-mbql5 as a parent and move the :h2-mbql5 methods below to just :h2.
+(driver/register! :h2-mbql5, :parent #{:h2 :sql-mbql5})
+
;;; this will prevent the H2 driver from showing up in the list of options when adding a new Database.
(defmethod driver/superseded-by :h2 [_driver] :deprecated)
@@ -94,6 +100,10 @@
[driver [_ arg pattern]]
[:regexp_substr (sql.qp/->honeysql driver arg) (sql.qp/->honeysql driver pattern)])
+(defmethod sql.qp/->honeysql [:h2-mbql5 :regex-match-first]
+ [driver [_ _opts arg pattern]]
+ [:regexp_substr (sql.qp/->honeysql driver arg) (sql.qp/->honeysql driver pattern)])
+
(defmethod driver/connection-properties :h2
[_]
(->>
@@ -419,6 +429,10 @@
[driver [_ field]]
[:log10 (sql.qp/->honeysql driver field)])
+(defmethod sql.qp/->honeysql [:h2-mbql5 :log]
+ [driver [_ _opts field]]
+ [:log10 (sql.qp/->honeysql driver field)])
+
(defmethod sql.qp/->honeysql [:h2 ::sql.qp/expression-literal-text-value]
[driver [_ value]]
;; A literal text value gets compiled to a parameter placeholder like "?". H2 attempts to compile the prepared
@@ -430,6 +444,17 @@
(->> (sql.qp/->honeysql driver value)
(h2x/cast :text)))
+(defmethod sql.qp/->honeysql [:h2-mbql5 ::sql.qp/expression-literal-text-value]
+ [driver [_ _opts value]]
+ ;; A literal text value gets compiled to a parameter placeholder like "?". H2 attempts to compile the prepared
+ ;; statement immediately, presumably before the types of the params are known, and sometimes raises an "Unknown
+ ;; data type" error if it can't deduce the type. The recommended workaround is to insert an explicit CAST.
+ ;;
+ ;; https://linear.app/metabase/issue/QUE-726/
+ ;; https://github.com/h2database/h2database/issues/1383
+ (->> (sql.qp/->honeysql driver value)
+ (h2x/cast :text)))
+
(defn- datediff
"Like H2's `datediff` function but accounts for timestamps with time zones."
[unit x y]
diff --git a/src/metabase/driver/sql/parameters/substitution.clj b/src/metabase/driver/sql/parameters/substitution.clj
index 6f38a0cf3c3..30af4f42a00 100644
--- a/src/metabase/driver/sql/parameters/substitution.clj
+++ b/src/metabase/driver/sql/parameters/substitution.clj
@@ -9,6 +9,7 @@
(:refer-clojure :exclude [not-empty mapv])
(:require
[clojure.string :as str]
+ [medley.core :as m]
[metabase.driver :as driver]
[metabase.driver-api.core :as driver-api]
^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.driver.common.parameters :as params]
@@ -265,8 +266,14 @@
{:replacement-snippet snippet
:prepared-statement-args args}))
-(mu/defn- field->clause :- driver-api/mbql.schema.field
- [field other-opts]
+(defmulti field->clause
+ "Returns an MBQL field clause with the given options."
+ {:added "0.61.0" :arglists '([driver field other-opts])}
+ driver/dispatch-on-initialized-driver
+ :hierarchy #'driver/hierarchy)
+
+(mu/defmethod field->clause :default :- driver-api/mbql.schema.field
+ [_driver field other-opts]
(driver-api/normalize
driver-api/mbql.schema.field
[:field
@@ -276,7 +283,7 @@
::compiling-field-filter? true}
other-opts)]))
-(mu/defn- field->field-filter-clause :- driver-api/mbql.schema.field
+(mu/defn- field->field-filter-clause :- [:or driver-api/mbql.schema.field :mbql.clause/field]
[driver :- :keyword
field :- driver-api/schema.metadata.column
param-type :- driver-api/schema.parameter.type
@@ -287,7 +294,7 @@
;; middleware would work either because we don't know what Field this parameter actually refers to until we resolve
;; the parameter. There's probably _some_ way to structure things that would make this "duplicate" call unneeded, but
;; I haven't figured out what that is yet
- (field->clause field {:temporal-unit (align-temporal-unit-with-param-type-and-value driver field param-type value)}))
+ (field->clause driver field (m/assoc-some {} :temporal-unit (align-temporal-unit-with-param-type-and-value driver field param-type value))))
(mu/defn- field->identifier :- driver-api/schema.common.non-blank-string
"Return an appropriate snippet to represent this `field` in SQL given its param type.
@@ -310,6 +317,45 @@
params/map->DateTimeRange
->datetime-replacement-snippet-info)))
+(defmulti to-clause
+ "Helper to dispatch to `params.ops/to-clause` for `:sql` or `qp.params.ops/to-clause` for `:sql-mbql5`."
+ {:added "0.61.0" :arglists '([driver param])}
+ driver/dispatch-on-initialized-driver
+ :hierarchy #'driver/hierarchy)
+
+(defmethod to-clause :default [_driver param] (params.ops/to-clause param))
+
+(defmulti desugar-filter-clause
+ "Helper to dispatch to `driver-api/desugar-filter-clause` for `:sql` or `lib/desugar-filter-clause` for `:sql-mbql5`."
+ {:added "0.61.0" :arglists '([driver filter-clause])}
+ driver/dispatch-on-initialized-driver
+ :hierarchy #'driver/hierarchy)
+
+(defmethod desugar-filter-clause :default [_driver filter-clause]
+ #_{:clj-kondo/ignore [:deprecated-var]}
+ (driver-api/desugar-filter-clause filter-clause))
+
+(defmulti wrap-value-literals-in-mbql
+ "Helper to dispatch to `driver-api/wrap-value-literals-in-mbql` for `:sql`
+ or `driver-api/wrap-value-literals-in-mbql5` for `:sql-mbql5`."
+ {:added "0.61.0" :arglists '([driver mbql])}
+ driver/dispatch-on-initialized-driver
+ :hierarchy #'driver/hierarchy)
+
+(defmethod wrap-value-literals-in-mbql :default
+ [_driver mbql]
+ #_{:clj-kondo/ignore [:deprecated-var]}
+ (driver-api/wrap-value-literals-in-mbql mbql))
+
+(defmulti date-string->filter
+ "Helper to dispatch to `params.dates/date-string->filter` for `:sql`
+ or `qp.params.dates/date-string->filter` for `:sql-mbql5`."
+ {:added "0.61.0" :arglists '([driver date-string field])}
+ driver/dispatch-on-initialized-driver
+ :hierarchy #'driver/hierarchy)
+
+(defmethod date-string->filter :default [_driver date-string field] (params.dates/date-string->filter date-string field))
+
(mu/defn- field-filter->replacement-snippet-info :- ParamSnippetInfo
"Return `[replacement-snippet & prepared-statement-args]` appropriate for a field filter parameter."
[driver {{param-type :type, value :value, :as params} :value, field :field, :as field-filter}]
@@ -323,18 +369,18 @@
(params.ops/operator? param-type)
#_{:clj-kondo/ignore [:deprecated-var]}
(->> (assoc params :target [:dimension (field->field-filter-clause driver field param-type value)])
- params.ops/to-clause
- driver-api/desugar-filter-clause
- driver-api/wrap-value-literals-in-mbql
+ (to-clause driver)
+ (desugar-filter-clause driver)
+ (wrap-value-literals-in-mbql driver)
->honeysql
(honeysql->replacement-snippet-info driver))
(params.dates/exclusion-date-type param-type value)
(let [field-clause (field->field-filter-clause driver field param-type value)]
#_{:clj-kondo/ignore [:deprecated-var]}
- (->> (params.dates/date-string->filter value field-clause)
- driver-api/desugar-filter-clause
- driver-api/wrap-value-literals-in-mbql
+ (->> (date-string->filter driver value field-clause)
+ (desugar-filter-clause driver)
+ (wrap-value-literals-in-mbql driver)
->honeysql
(honeysql->replacement-snippet-info driver)))
@@ -361,7 +407,7 @@
[driver field alias replacement-snippet-info]
(if (str/blank? alias)
replacement-snippet-info
- (let [[old-name] (->> (field->clause field nil)
+ (let [[old-name] (->> (field->clause driver field nil)
(sql.qp/->honeysql driver)
(sql.qp/format-honeysql driver))]
(update replacement-snippet-info :replacement-snippet str/replace old-name alias))))
@@ -403,12 +449,10 @@
(defmethod ->replacement-snippet-info [:sql TemporalUnit]
[driver {:keys [value field alias]}]
- (let [replacement-snippet-info
- (honeysql->replacement-snippet-info
- driver
- (sql.qp/->honeysql driver
- (field->clause field (when (not= value params/no-value)
- {:temporal-unit (keyword value)}))))]
+ (let [replacement-snippet-info (->> (field->clause driver field (when (not= value params/no-value)
+ {:temporal-unit (keyword value)}))
+ (sql.qp/->honeysql driver)
+ (honeysql->replacement-snippet-info driver))]
(replace-alias driver field alias replacement-snippet-info)))
(defmethod ->replacement-snippet-info [:sql ReferencedTableQuery]
diff --git a/src/metabase/driver/sql/query_processor.clj b/src/metabase/driver/sql/query_processor.clj
index ef18467cf64..f257af50ea3 100644
--- a/src/metabase/driver/sql/query_processor.clj
+++ b/src/metabase/driver/sql/query_processor.clj
@@ -14,6 +14,7 @@
[metabase.driver.sql.query-processor.deprecated :as sql.qp.deprecated]
[metabase.util :as u]
[metabase.util.date-2 :as u.date]
+ [metabase.util.experiment :as experiment]
[metabase.util.honey-sql-2 :as h2x]
[metabase.util.i18n :refer [tru]]
[metabase.util.log :as log]
@@ -300,6 +301,21 @@
(sql/register-fn! ::compiled #'format-compiled)
+(defmulti mbql-clause-with-opts
+ "Returns an MBQL clause in the desired MBQL format of the driver."
+ {:added "0.61.0" :arglists '([driver tag opts & args])}
+ driver/dispatch-on-initialized-driver
+ :hierarchy #'driver/hierarchy)
+
+(defmethod mbql-clause-with-opts :sql
+ [_driver tag opts & args]
+ (cond-> (into [tag] args) opts (conj opts)))
+
+(defn mbql-clause
+ "Wrapper around `mbql-clause-with-opts` for clauses that don't have or need options."
+ [driver tag & args]
+ (apply mbql-clause-with-opts driver tag nil args))
+
(defmulti current-datetime-honeysql-form
"HoneySQL form that should be used to get the current `datetime` (or equivalent). Defaults to `:%now`. Should ideally
include the database type info on the form (ex: via [[h2x/with-type-info]])."
@@ -332,7 +348,7 @@
(defmethod date [:sql :week-of-year]
[driver _ expr]
;; Some DBs truncate when doing integer division, therefore force float arithmetic
- (->honeysql driver [:ceil (compiled (h2x// (date driver :day-of-year (date driver :week expr)) 7.0))]))
+ (->honeysql driver (mbql-clause driver :ceil (compiled (h2x// (date driver :day-of-year (date driver :week expr)) 7.0)))))
(defmethod date [:sql :month-of-year] [_driver _ expr] (h2x/month expr))
(defmethod date [:sql :quarter-of-year] [_driver _ expr] (h2x/quarter expr))
@@ -394,7 +410,7 @@
(days-till-start-of-first-full-week driver honeysql-expr))
total-full-week-days (h2x/- (date driver :day-of-year honeysql-expr)
days-till-start-of-first-full-week)
- total-full-weeks (->honeysql driver [:ceil (compiled (h2x// total-full-week-days 7.0))])]
+ total-full-weeks (->honeysql driver (mbql-clause driver :ceil (compiled (h2x// total-full-week-days 7.0))))]
(->integer driver (h2x/+ 1 total-full-weeks))))
;; ISO8501 consider the first week of the year is the week that contains the 1st Thursday and week starts on Monday.
@@ -700,7 +716,7 @@
:else value)
(->honeysql driver value))))
-(defn- literal-text-value?
+(defn- literal-text-value?*
[[_ value {base-type :base_type effective-type :effective_type} :as clause]]
(and (driver-api/is-clause? :value clause)
(string? value)
@@ -710,6 +726,24 @@
(isa? (or effective-type base-type)
:type/Text))))
+(defn- literal-text-value?
+ [clause]
+ (literal-text-value?*
+ (driver-api/match-one clause
+ [tag (opts :guard :lib/uuid) value] ;; mbql5
+ [tag value {:base_type (:base-type opts) :effective_type (:effective-type opts)}]
+ _ clause)))
+
+(defmulti expression-by-name
+ "Gets an expression from a query or stage (`*inner-query`) by name."
+ {:added "0.61.0" :arglists '([driver inner-query expression-name])}
+ driver/dispatch-on-initialized-driver
+ :hierarchy #'driver/hierarchy)
+
+(defmethod expression-by-name :sql
+ [_driver inner-query expression-name]
+ (driver-api/expression-with-name inner-query expression-name))
+
(defn apply-temporal-bucketing
"Apply temporal bucketing for the `:temporal-unit` in the options of a `:field` clause; return a new HoneySQL form that
buckets `honeysql-form` appropriately."
@@ -720,13 +754,13 @@
[driver [_ expression-name opts :as _clause]]
(let [source-table (get opts driver-api/qp.add.source-table)
source-alias (get opts driver-api/qp.add.source-alias)
- expression-definition (driver-api/expression-with-name *inner-query* expression-name)]
+ expression-definition (expression-by-name driver *inner-query* expression-name)]
(cond->>
(->honeysql driver (cond (= source-table driver-api/qp.add.source)
(apply h2x/identifier :field source-query-alias source-alias)
(literal-text-value? expression-definition)
- [::expression-literal-text-value expression-definition]
+ (mbql-clause driver ::expression-literal-text-value expression-definition)
;; Handle raw string literals (not wrapped in :value) - needed for
;; expression definitions that are just string literals, e.g. from
@@ -734,8 +768,8 @@
;; the string becomes a parameter placeholder without type info,
;; which some databases (like H2) can't handle.
(string? expression-definition)
- [::expression-literal-text-value
- [:value expression-definition {:base_type :type/Text}]]
+ (mbql-clause driver ::expression-literal-text-value
+ (mbql-clause-with-opts driver :value {:base_type :type/Text} expression-definition))
:else
expression-definition))
@@ -931,7 +965,7 @@
;; preserve metadata attached to the original field clause, for example BigQuery temporal type information.
identifier (-> (apply h2x/identifier :field
(concat source-table-aliases
- (->honeysql driver [::nfc-path source-nfc-path])
+ (->honeysql driver (mbql-clause driver ::nfc-path source-nfc-path))
parent-names
[source-alias]))
(with-meta (meta field-clause)))
@@ -1037,41 +1071,64 @@
(or (clause-pred (first form))
(m/find-first (partial contains-clause? clause-pred) (rest form))))))
+(defmulti over-order-by->honeysql
+ "Returns the HoneySQL for an order by clause in the over clause of a window function."
+ {:added "0.61.0" :arglists '([driver aggregations order-by])}
+ driver/dispatch-on-initialized-driver
+ :hierarchy #'driver/hierarchy)
+
+(defmethod over-order-by->honeysql :sql
+ [driver aggregations [direction expr]]
+ (if (aggregation? expr)
+ (let [[_aggregation index] expr
+ agg (unwrap-aggregation-option (aggregations index))]
+ (when-not (contains-clause? #{:cum-count :cum-sum :offset} agg)
+ [(->honeysql driver agg) direction]))
+ [(->honeysql driver expr) direction]))
+
(defn- over-order-bys
"Returns a vector containing the `aggregations` specified by `order-bys` compiled to
honeysql expressions for `driver` suitable for ordering in the over clause of a window function."
[driver aggregations order-bys]
(let [aggregations (vec aggregations)]
(into []
- (keep (fn [[direction expr]]
- (if (aggregation? expr)
- (let [[_aggregation index] expr
- agg (unwrap-aggregation-option (aggregations index))]
- (when-not (contains-clause? #{:cum-count :cum-sum :offset} agg)
- [(->honeysql driver agg) direction]))
- [(->honeysql driver expr) direction])))
+ (keep (fn [order-by]
+ (over-order-by->honeysql driver aggregations order-by)))
order-bys)))
+(defn- remapped-order-by? [order-by]
+ (driver-api/qp.util.transformations.nest-breakouts.externally-remapped-field
+ (driver-api/match-one order-by
+ [_dir (_opts :guard :lib/uuid) [_ (opts :guard :lib/uuid) _name]] opts ;; mbql5
+ [_dir [_ _name opts]] opts)))
+
+(defn- remapped-breakout? [breakout]
+ (driver-api/qp.util.transformations.nest-breakouts.externally-remapped-field
+ (driver-api/match-one breakout
+ [_ (opts :guard :lib/uuid) _name] opts ;; mbql5
+ [_ _name opts] opts)))
+
+(defmulti breakout-options-index
+ "Returns the index of options in a breakout clause."
+ {:added "0.61.0", :arglists '([driver])}
+ driver/dispatch-on-initialized-driver
+ :hierarchy #'driver/hierarchy)
+
+(defmethod breakout-options-index :sql [_driver] 2)
+
(defn- window-aggregation-over-expr-for-query-with-breakouts
"Order by the first breakout, then partition by all the other ones. See #42003 and
https://metaboat.slack.com/archives/C05MPF0TM3L/p1714084449574689 for more info."
[driver inner-query]
- (let [breakouts (into []
- (remove
- (comp driver-api/qp.util.transformations.nest-breakouts.externally-remapped-field
- #(nth % 2)))
- (:breakout inner-query))
+ (let [breakouts (into [] (remove remapped-breakout?) (:breakout inner-query))
group-bys (:group-by (apply-top-level-clause driver :breakout {} inner-query))
- finest-temp-breakout (driver-api/finest-temporal-breakout-index breakouts 2)
+ opts-idx (breakout-options-index driver)
+ finest-temp-breakout (driver-api/finest-temporal-breakout-index breakouts opts-idx)
partition-exprs (when (> (count breakouts) 1)
(if finest-temp-breakout
(m/remove-nth finest-temp-breakout group-bys)
(butlast group-bys)))
- order-bys (remove
- (comp driver-api/qp.util.transformations.nest-breakouts.externally-remapped-field
- #(nth % 2)
- second)
- (:order-by inner-query))
+ order-bys (remove remapped-order-by? (:order-by inner-query))
order-bys (over-order-bys driver
(:aggregation inner-query)
order-bys)]
@@ -1156,7 +1213,7 @@
[driver [_cum-count expr-or-nil]]
;; a cumulative count with no breakouts doesn't really mean anything, just compile it as a normal count.
(if (empty? (:breakout *inner-query*))
- (->honeysql driver [:count expr-or-nil])
+ (->honeysql driver (mbql-clause driver :count expr-or-nil))
(cumulative-aggregation-over-rows
driver
[:sum (if expr-or-nil
@@ -1178,7 +1235,7 @@
[driver [_cum-sum expr]]
;; a cumulative sum with no breakouts doesn't really mean anything, just compile it as a normal sum.
(if (empty? (:breakout *inner-query*))
- (->honeysql driver [:sum expr])
+ (->honeysql driver (mbql-clause driver :sum expr))
(cumulative-aggregation-over-rows
driver
[:sum [:sum (->honeysql driver expr)]])))
@@ -1198,6 +1255,11 @@
(defn- interval? [expr]
(driver-api/is-clause? :interval expr))
+(defn- normalize-interval [interval]
+ (driver-api/match-one interval
+ [tag (_opts :guard :lib/uuid) amount unit] [tag amount unit] ;; mbql5
+ _ interval))
+
(defmethod ->honeysql [:sql :+]
[driver [_ & args]]
(if (some interval? args)
@@ -1205,7 +1267,7 @@
(reduce (fn [hsql-form [_ amount unit]]
(add-interval-honeysql-form driver hsql-form amount unit))
(->honeysql driver field)
- intervals)
+ (map normalize-interval intervals))
(throw (ex-info "Summing intervals is not supported" {:args args})))
(into [:+]
(map (partial ->honeysql driver))
@@ -1227,7 +1289,7 @@
;; We are adding negative amount. Inspired by `->honeysql [:sql :datetime-subtract]`.
(add-interval-honeysql-form driver hsql-form (- amount) unit))
(->honeysql driver first-arg)
- other-args)
+ (map normalize-interval other-args))
(into [:-]
(map (partial ->honeysql driver))
args)))
@@ -1280,20 +1342,20 @@
(defmethod ->honeysql [:sql :sum-where]
[driver [_ arg pred]]
- (->honeysql driver [:sum [:case [[pred arg]] {:default 0.0}]]))
+ (->honeysql driver (mbql-clause driver :sum (mbql-clause driver :case [[pred arg]] {:default 0.0}))))
(defmethod ->honeysql [:sql :count-where]
[driver [_ pred]]
- (->honeysql driver [:sum-where 1 pred]))
+ (->honeysql driver (mbql-clause driver :sum-where 1 pred)))
(defmethod ->honeysql [:sql :share]
[driver [_ pred]]
- [:/ (->honeysql driver [:count-where pred]) :%count.*])
+ [:/ (->honeysql driver (mbql-clause driver :count-where pred)) :%count.*])
(defmethod ->honeysql [:sql :distinct-where]
[driver [_ arg pred]]
[::h2x/distinct-count
- (->honeysql driver [:case [[pred arg]]])])
+ (->honeysql driver (mbql-clause driver :case [[pred arg]]))])
(defmethod ->honeysql [:sql :trim]
[driver [_ arg]]
@@ -1399,11 +1461,11 @@
(defmethod ->honeysql [:sql :text]
[driver [_ value]]
- (->honeysql driver [::cast-to-text value]))
+ (->honeysql driver (mbql-clause driver ::cast-to-text value)))
(defmethod ->honeysql [:sql :today]
[driver [_]]
- (->honeysql driver [:date [:now]]))
+ (->honeysql driver (mbql-clause driver :date (mbql-clause driver :now))))
(mu/defmethod ->honeysql [:sql :relative-datetime] :- some?
[driver [_ amount unit]]
@@ -1455,8 +1517,12 @@
Optional third parameter `unique-name-fn` is no longer used as of 0.42.0."
([driver :- :keyword
- [clause-type id-or-name opts] :- vector?]
- (let [desired-alias (or (get opts driver-api/qp.add.desired-alias)
+ clause :- vector?]
+ (let [[clause-type id-or-name opts] (driver-api/match-one clause
+ [clause-type (opts :guard :lib/uuid) id-or-name] ;; mbql5
+ [clause-type id-or-name opts]
+ _ clause)
+ desired-alias (or (get opts driver-api/qp.add.desired-alias)
;; fallback behavior for anyone using SQL QP functions directly without including the stuff
;; from [[metabase.query-processor.util.add-alias-info]]. We should probably disallow this
;; going forward because it is liable to break
@@ -1536,27 +1602,31 @@
;;
;; See #17536 and #18742
+(defn- force-using-column-alias-opts
+ [opts is-breakout]
+ (cond-> (assoc opts
+ driver-api/qp.add.source-alias (get opts driver-api/qp.add.desired-alias)
+ driver-api/qp.add.source-table driver-api/qp.add.none
+ ;; this key will tell the SQL QP not to apply casting here either.
+ :qp/ignore-coercion true
+ ;; used to indicate that this is a forced alias
+ ::forced-alias true)
+ ;; don't want to do temporal bucketing or binning inside the order by only.
+ ;; That happens inside the `SELECT`
+ ;; (#22831) however, we do want it in breakout
+ (not is-breakout) (dissoc :temporal-unit :binning)))
+
(defn rewrite-fields-to-force-using-column-aliases
"Rewrite `:field` clauses to force them to use the column alias regardless of where they appear."
([form]
(rewrite-fields-to-force-using-column-aliases form {:is-breakout false}))
([form {is-breakout :is-breakout}]
- (driver-api/replace
- form
+ (driver-api/replace form
+ [:field (opts :guard :lib/uuid) id-or-name] ;; mbql5
+ [:field (force-using-column-alias-opts opts is-breakout) id-or-name]
+
[:field id-or-name opts]
- [:field id-or-name (cond-> opts
- true
- (assoc driver-api/qp.add.source-alias (get opts driver-api/qp.add.desired-alias)
- driver-api/qp.add.source-table driver-api/qp.add.none
- ;; this key will tell the SQL QP not to apply casting here either.
- :qp/ignore-coercion true
- ;; used to indicate that this is a forced alias
- ::forced-alias true)
- ;; don't want to do temporal bucketing or binning inside the order by only.
- ;; That happens inside the `SELECT`
- ;; (#22831) however, we do want it in breakout
- (not is-breakout)
- (dissoc :temporal-unit :binning))])))
+ [:field id-or-name (force-using-column-alias-opts opts is-breakout)])))
;;; +----------------------------------------------------------------------------------------------------------------+
;;; | Clause Handlers |
@@ -1564,11 +1634,21 @@
;;; -------------------------------------------------- aggregation ---------------------------------------------------
+(defmulti aggregation-name
+ "Returns the name of an aggregation clause."
+ {:added "0.61.0" :arglists '([driver inner-query ag-clause])}
+ driver/dispatch-on-initialized-driver
+ :hierarchy #'driver/hierarchy)
+
+(defmethod aggregation-name :sql
+ [_driver inner-query ag-clause]
+ (driver-api/aggregation-name inner-query ag-clause))
+
(defmethod apply-top-level-clause [:sql :aggregation]
[driver _top-level-clause honeysql-form {aggregations :aggregation, :as inner-query}]
(let [honeysql-ags (vec (for [ag aggregations
:let [ag-expr (->honeysql driver ag)
- ag-name (driver-api/aggregation-name inner-query ag)
+ ag-name (aggregation-name driver inner-query ag)
ag-alias (->honeysql driver (h2x/identifier
:field-alias
(driver/escape-alias driver ag-name)))]]
@@ -1623,7 +1703,9 @@
[:or
[:and driver-api/mbql.schema.value
[:fn {:error/message "string value"} #(string? (second %))]]
- driver-api/mbql.schema.FieldOrExpressionDef])
+ driver-api/mbql.schema.FieldOrExpressionDef
+ ;; mbql5
+ [:ref :metabase.lib.schema.expression/string]])
(defmulti escape-like-pattern
"Handle escaping a literal string into a `LIKE` clause pattern which will match literally.
@@ -1661,6 +1743,14 @@
[_driver like-rhs-honeysql]
[:escape like-rhs-honeysql [:inline "\\"]])
+(defmulti clause-value-idx
+ "Returns the index of the value in a value clause."
+ {:added "0.61.0" :arglists '([driver])}
+ driver/dispatch-on-initialized-driver
+ :hierarchy #'driver/hierarchy)
+
+(defmethod clause-value-idx :sql [_driver] 1)
+
(mu/defn- generate-pattern
"Generate pattern to match against in like clause. Lowercasing for case insensitive matching also happens here."
[driver
@@ -1669,47 +1759,49 @@
post
{:keys [case-sensitive] :or {case-sensitive true} :as _options}]
(if (= :value type)
- (->> (update arg 1 #(cond-> (str pre (escape-like-pattern driver %) post)
- (not case-sensitive) u/lower-case-en))
+ (->> (update arg (clause-value-idx driver) #(cond-> (str pre (escape-like-pattern driver %) post)
+ (not case-sensitive) u/lower-case-en))
(->honeysql driver)
(transform-literal-like-pattern-honeysql driver))
- (let [expr (->honeysql driver (into [:concat] (remove nil?) [pre arg post]))]
+ (let [expr (->honeysql driver (apply mbql-clause driver (into [:concat] (remove nil?) [pre arg post])))]
(if case-sensitive
expr
[:lower expr]))))
(defn- uuid-field?
[x]
- (and (driver-api/mbql-clause? x)
- (isa? (or (:effective-type (get x 2))
- (let [field-id (second x)]
+ (let [[opts field-id] (driver-api/match-one x
+ [:field (opts :guard :lib/uuid) field-id] [opts field-id] ;; mbql5
+ [:field field-id opts] [opts field-id])]
+ (and (driver-api/mbql-clause? x)
+ (isa? (or (:effective-type opts)
(when (pos-int? field-id)
(let [{:keys [base-type effective-type]}
(driver-api/field (driver-api/metadata-provider) field-id)]
- (or effective-type base-type))))
- (:base-type (get x 2)))
- :type/UUID)))
+ (or effective-type base-type)))
+ (:base-type opts))
+ :type/UUID))))
(mu/defn- maybe-cast-uuid-for-equality
"For := and :!=. Comparing UUID fields against non-uuid values requires casting."
[driver field arg]
(if (and (uuid-field? field)
- ;; If the arg is a uuid we are happy especially for joins (#46558)
+ ;; If the arg is a uuid we are happy especially for joins (#46558)
(not (uuid-field? arg))
- ;; If we could not convert the arg to a UUID then we have to cast the Field.
- ;; This will not hit indexes, but then we're passing an arg that can only be compared textually.
+ ;; If we could not convert the arg to a UUID then we have to cast the Field.
+ ;; This will not hit indexes, but then we're passing an arg that can only be compared textually.
(not (uuid? (->honeysql driver arg)))
- ;; Check for inlined values
+ ;; Check for inlined values
(not (= (:database-type (h2x/type-info (->honeysql driver arg))) "uuid")))
- [::cast-to-text field]
+ (mbql-clause driver ::cast-to-text field)
field))
(mu/defn maybe-cast-uuid-for-text-compare
"For :contains, :starts-with, and :ends-with.
Comparing UUID fields against with these operations requires casting as the right side will have `%` for `LIKE` operations."
- [field]
+ [driver field]
(if (uuid-field? field)
- [::cast-to-text field]
+ (mbql-clause driver ::cast-to-text field)
field))
(defmethod ->honeysql [:sql ::cast]
@@ -1721,7 +1813,7 @@
;; Oracle does not support text type,
;; sqlserver limits varchar to 30 in casts,
;; athena cannot cast uuid to bounded varchars
- (->honeysql driver [::cast expr "text"]))
+ (->honeysql driver (mbql-clause driver ::cast expr "text")))
(defmethod ->honeysql [:sql ::expression-literal-text-value]
[driver [_ value]]
@@ -1737,17 +1829,17 @@
(defmethod ->honeysql [:sql :starts-with]
[driver [_ field arg options]]
- (like-clause (->honeysql driver (maybe-cast-uuid-for-text-compare field))
+ (like-clause (->honeysql driver (maybe-cast-uuid-for-text-compare driver field))
(generate-pattern driver nil arg "%" options) options))
(defmethod ->honeysql [:sql :contains]
[driver [_ field arg options]]
- (like-clause (->honeysql driver (maybe-cast-uuid-for-text-compare field))
+ (like-clause (->honeysql driver (maybe-cast-uuid-for-text-compare driver field))
(generate-pattern driver "%" arg "%" options) options))
(defmethod ->honeysql [:sql :ends-with]
[driver [_ field arg options]]
- (like-clause (->honeysql driver (maybe-cast-uuid-for-text-compare field))
+ (like-clause (->honeysql driver (maybe-cast-uuid-for-text-compare driver field))
(generate-pattern driver "%" arg nil options) options))
(defn- parent-honeysql-col-base-type-map
@@ -1816,9 +1908,17 @@
[:= (->honeysql driver field-arg) nil]]
honeysql-clause)))
+(defn- unwrap-value-literal
+ "Extract value literal from `:value` form or returns form as is if not a `:value` form."
+ [maybe-value-form]
+ (driver-api/match-one maybe-value-form
+ [:value (opts :guard :lib/uuid) x & _] x ;; mbql5
+ [:value x & _] x
+ _ maybe-value-form))
+
(defmethod ->honeysql [:sql :!=]
[driver [_ field value]]
- (if (nil? (driver-api/unwrap-value-literal value))
+ (if (nil? (unwrap-value-literal value))
[:not= (->honeysql driver (maybe-cast-uuid-for-equality driver field value)) (->honeysql driver value)]
(correct-null-behaviour driver [:not= (maybe-cast-uuid-for-equality driver field value) value])))
@@ -1875,7 +1975,7 @@
:else
(->honeysql driver (driver-api/table (driver-api/metadata-provider) source-table))))
-(def ^:private HoneySQLJoin
+(def HoneySQLJoin
"Schema for HoneySQL for a single JOIN. Used to validate that our join-handling code generates correct clauses."
[:tuple
;;join source and alias
@@ -1892,8 +1992,7 @@
(let [join-alias ((some-fn driver-api/qp.add.alias :alias) join)]
(assert (string? join-alias))
[[(join-source driver join)
- (let [table-alias (->honeysql driver (h2x/identifier :table-alias join-alias))]
- [table-alias])]
+ [(->honeysql driver (h2x/identifier :table-alias join-alias))]]
(->honeysql driver condition)]))
(defn- apply-joins-honey-sql-2
@@ -1972,7 +2071,7 @@
{:source-table 0, :breakout 1, ...}"
(into {} (map-indexed
#(vector %2 %1)
- [:source-table :breakout :aggregation :fields :filter :joins :order-by :page :limit])))
+ [:source-table :breakout :aggregation :fields :filter :filters :joins :order-by :page :limit])))
(defn- query->keys-in-application-order
"Return the keys present in an MBQL `inner-query` in the order they should be processed."
@@ -2068,6 +2167,18 @@
(declare apply-clauses)
+(defn needs-cte-for-duplicate-cols?
+ "Determines if the source query has ambiguous columns names"
+ [source-metadata]
+ (let [source-aliases (mapv :lib/source-column-alias source-metadata)
+ desired-aliases (mapv :lib/desired-column-alias source-metadata)]
+ (and (seq desired-aliases)
+ (> (count desired-aliases) 1)
+ (apply distinct? desired-aliases)
+ (every? string? desired-aliases)
+ (> (count source-aliases) 1)
+ (not (apply distinct? source-aliases)))))
+
(defn- apply-source-query
"Handle a `:source-query` clause by adding a recursive `SELECT` or native query. If the source query has ambiguous
column names, use a `WITH` statement to rename the source columns. At the time of this writing, all source queries
@@ -2084,18 +2195,10 @@
:else
(apply-clauses driver {} source-query))
- source-aliases (mapv :lib/source-column-alias source-metadata)
- desired-aliases (mapv :lib/desired-column-alias source-metadata)
- duplicate-source-aliases? (and (> (count source-aliases) 1)
- (not (apply distinct? source-aliases)))
- needs-columns? (and (seq desired-aliases)
- (> (count desired-aliases) 1)
- duplicate-source-aliases?
- (apply distinct? desired-aliases)
- (every? string? desired-aliases))]
+ desired-aliases (mapv :lib/desired-column-alias source-metadata)]
(merge
honeysql-form
- (if needs-columns?
+ (if (needs-cte-for-duplicate-cols? source-metadata)
;; HoneySQL cannot expand [::h2x/identifier :table "__mb_source"] in the with alias.
;; This is ok since we control the alias.
{:with [[[source-query-alias {:columns (mapv #(h2x/identifier :field %) desired-aliases)}]
@@ -2122,7 +2225,7 @@
to use this function in your own implementations of [[driver/mbql->native]], if you want to apply changes to the
same version of the query that we will ultimately be compiling.
- Wants a `:lib/query` MBQL 5 query as input. Always returns an MBQL 4 **inner query**, for historical reasons."
+ Wants a `:lib/query` MBQL 5 query as input. Returns an MBQL 4 **inner query**, for historical reasons, or MBQL 5 **stages**."
{:changelog-test/ignore true, :arglists '([driver mbql5-query]), :added "0.42.0"}
driver/dispatch-on-initialized-driver
:hierarchy #'driver/hierarchy)
@@ -2136,15 +2239,22 @@
driver-api/->legacy-MBQL
:query))
-(mu/defn mbql->honeysql :- [:or :map [:tuple [:= :inline] :map]]
- "Build the HoneySQL form we will compile to SQL and execute."
- [driver :- :keyword
- query :- :map]
+(defmulti compile-mbql
+ "Compiles an MBQL inner query to HoneySQL."
+ {:added "0.61.0" :arglists '([driver mbql])}
+ driver/dispatch-on-initialized-driver
+ :hierarchy #'driver/hierarchy)
+
+(defmethod compile-mbql :sql
+ [driver query]
+ (apply-clauses driver {} query))
+
+(defn- mbql->honeysql* [driver query]
(if (:lib/type query)
(binding [driver/*driver* driver]
(let [inner-query (preprocess driver query)]
(log/tracef "Compiling MBQL query\n%s" (u/pprint-to-str 'magenta inner-query))
- (u/prog1 (apply-clauses driver {} inner-query)
+ (u/prog1 (compile-mbql driver inner-query)
(log/debugf "\nHoneySQL Form: %s\n%s" (u/emoji "🍯") (u/pprint-to-str 'cyan <>))
(driver-api/debug> (list '🍯 <>)))))
@@ -2156,6 +2266,31 @@
mbql5-query (driver-api/query-from-legacy-inner-query metadata-provider database-id inner-query)]
(recur driver mbql5-query))))
+(def ^:private experimental-mbql5-drivers {:h2 :h2-mbql5})
+
+(defn- mbql5-experiment-report
+ [driver experimental-driver query]
+ (fn [{:keys [match? candidate-outcome control-outcome] :as result}]
+ (when-let [report-fn @experiment/default-report-fn]
+ (report-fn result))
+ (when-not match?
+ (log/with-context {:experimental-mbql5-driver :mismatch
+ :driver driver
+ :experimental-driver experimental-driver}
+ (log/warnf "MBQL5 experiment mismatch:\nQuery: %s\nControl result: %s\nCandidate result: %s"
+ query control-outcome candidate-outcome)))))
+
+(mu/defn mbql->honeysql :- [:or :map [:tuple [:= :inline] :map]]
+ "Build the HoneySQL form we will compile to SQL and execute."
+ [driver :- :keyword
+ query :- :map]
+ (if-let [experimental-driver (experimental-mbql5-drivers driver)]
+ (experiment/experiment {:name :experimental-mbql5-driver
+ :report-fn (mbql5-experiment-report driver experimental-driver query)}
+ (mbql->honeysql* driver query)
+ (mbql->honeysql* experimental-driver query))
+ (mbql->honeysql* driver query)))
+
;;;; MBQL -> Native
(mu/defn mbql->native :- [:map
diff --git a/src/metabase/driver/sql_jdbc/actions.clj b/src/metabase/driver/sql_jdbc/actions.clj
index f8981c1e166..219bcd17baa 100644
--- a/src/metabase/driver/sql_jdbc/actions.clj
+++ b/src/metabase/driver/sql_jdbc/actions.clj
@@ -124,7 +124,7 @@
(if-let [sql-type (type->sql-type base-type)]
(h2x/cast sql-type value)
(try
- (sql.qp/->honeysql driver [:value value field])
+ (sql.qp/->honeysql driver (sql.qp/mbql-clause driver :value value field))
(catch Exception e
(throw (ex-info (str "column cast failed: " (pr-str col-name))
{:column col-name
diff --git a/src/metabase/driver/sql_mbql5.clj b/src/metabase/driver/sql_mbql5.clj
new file mode 100644
index 00000000000..85dfc0b928e
--- /dev/null
+++ b/src/metabase/driver/sql_mbql5.clj
@@ -0,0 +1,199 @@
+(ns metabase.driver.sql-mbql5
+ "An abstract driver to allow for opt-in MBQL5 compilation."
+ (:refer-clojure :exclude [mapv get-in])
+ (:require
+ [honey.sql.helpers :as sql.helpers]
+ [medley.core :as m]
+ [metabase.driver :as driver]
+ [metabase.driver-api.core :as driver-api]
+ [metabase.driver.sql.parameters.substitution :as sql.params.substitution]
+ [metabase.driver.sql.query-processor :as sql.qp]
+ [metabase.lib.core :as lib]
+ [metabase.lib.options :as lib.options]
+ [metabase.lib.schema.common :as lib.schema.common]
+ [metabase.lib.util :as lib.util]
+ [metabase.query-processor.parameters.dates :as qp.params.dates]
+ [metabase.query-processor.parameters.operators :as qp.params.ops]
+ [metabase.query-processor.util.add-alias-info :as add]
+ [metabase.util.honey-sql-2 :as h2x]
+ [metabase.util.malli :as mu]
+ [metabase.util.performance :refer [mapv get-in]]))
+
+(driver/register! :sql-mbql5, :parent :sql, :abstract? true)
+
+(defmethod sql.qp/preprocess :sql-mbql5
+ [_driver mbql5-query]
+ (-> mbql5-query
+ driver-api/nest-breakouts-in-stages-with-window-aggregation
+ driver-api/nest-expressions
+ driver-api/add-alias-info
+ :stages))
+
+(defn- stage->honeysql [driver stage]
+ (cond
+ (:persisted-info/native stage)
+ (sql.qp/sql-source-query (:persisted-info/native stage) nil)
+
+ (= (:lib/type stage) :mbql.stage/native)
+ (sql.qp/sql-source-query (:native stage) (:params stage))
+
+ :else
+ (binding [sql.qp/*inner-query* stage]
+ (#'sql.qp/apply-top-level-clauses driver {} stage))))
+
+(defn- stages->honeysql [driver stages]
+ (first
+ (reduce
+ (fn [[prev-hsql prev-stage] stage]
+ (let [stage-hsql (stage->honeysql driver stage)]
+ (if prev-hsql
+ (let [table-alias (sql.qp/->honeysql driver (h2x/identifier :table-alias sql.qp/source-query-alias))
+ columns-metadata (get-in prev-stage [:lib/stage-metadata :columns])
+ desired-aliases (mapv :lib/desired-column-alias columns-metadata)
+ cur-hsql (cond-> (assoc stage-hsql :from [[prev-hsql [table-alias]]])
+ (sql.qp/needs-cte-for-duplicate-cols? columns-metadata)
+ (assoc :with [[[sql.qp/source-query-alias {:columns (mapv #(h2x/identifier :field %) desired-aliases)}]
+ prev-hsql]]
+ :from [[table-alias]])
+
+ ;; Clear the default select if it's just :* so add-default-select can recompute it
+ (= (:select stage-hsql) [[:*]])
+ (dissoc :select))]
+ [(#'sql.qp/add-default-select driver cur-hsql) stage])
+ [stage-hsql stage])))
+ [nil nil]
+ stages)))
+
+(defmethod sql.qp/compile-mbql :sql-mbql5
+ [driver stages]
+ (stages->honeysql driver stages))
+
+(defn- predicates->honeysql [driver predicates]
+ (let [predicates (mapv #(sql.qp/->honeysql driver %) predicates)]
+ (if (= 1 (count predicates))
+ (first predicates)
+ (into [:and] predicates))))
+
+(defmethod sql.qp/apply-top-level-clause [:sql-mbql5 :filters]
+ [driver _ honeysql-form {:keys [filters]}]
+ (sql.helpers/where honeysql-form (predicates->honeysql driver filters)))
+
+(defmethod sql.qp/join-source :sql-mbql5
+ [driver {:keys [stages]}]
+ (stages->honeysql driver stages))
+
+(mu/defmethod sql.qp/join->honeysql :sql-mbql5 :- sql.qp/HoneySQLJoin
+ [driver {:keys [conditions] :as join} :- [:ref :metabase.lib.schema.join/join]]
+ (let [join-alias ((some-fn driver-api/qp.add.alias :alias) join)]
+ (assert (string? join-alias))
+ [[(sql.qp/join-source driver join)
+ [(sql.qp/->honeysql driver (h2x/identifier :table-alias join-alias))]]
+ (predicates->honeysql driver conditions)]))
+
+(defn- agg-by-id [aggs agg-uuid]
+ (m/find-first #(= (lib.options/uuid %) agg-uuid) aggs))
+
+;; Should mirror the logic in `sql.qp/->honeysql [:sql :aggregation]`
+(defmethod sql.qp/->honeysql [:sql-mbql5 :aggregation]
+ [driver [_ _opts _agg-uuid :as clause]]
+ (driver-api/match-one clause
+ [:aggregation {driver-api/qp.add.desired-alias desired-alias} _agg-uuid]
+ (sql.qp/->honeysql driver (h2x/identifier :field-alias desired-alias))
+
+ [:aggregation {driver-api/qp.add.source-alias source-alias} _agg-uuid]
+ (sql.qp/->honeysql driver (h2x/identifier :field-alias source-alias))
+
+ [:aggregation {:lib/source-name source-name} _agg-uuid]
+ (sql.qp/->honeysql driver (h2x/identifier :field-alias source-name))
+
+ [:aggregation _opts agg-uuid]
+ (let [agg (agg-by-id (:aggregation sql.qp/*inner-query*) agg-uuid)]
+ (&recur agg))
+
+ [:distinct & _]
+ (sql.qp/->honeysql driver (h2x/identifier :field-alias :count))
+
+ [#{:+ :- :* :/} & _]
+ (sql.qp/->honeysql driver &match)
+
+ [:offset {:lib/source-name source-name} _expr _n]
+ (sql.qp/->honeysql driver (h2x/identifier :field-alias source-name))
+
+ [ag-type & _]
+ (sql.qp/->honeysql driver (h2x/identifier :field-alias ag-type))))
+
+(defmethod sql.qp/over-order-by->honeysql :sql-mbql5
+ [driver aggregations [direction _opts expr]]
+ (if (lib.util/clause-of-type? expr :aggregation)
+ (let [[_op _opts agg-uuid] expr
+ aggregation (agg-by-id aggregations agg-uuid)]
+ (when-not (#'sql.qp/contains-clause? #{:cum-count :cum-sum :offset} aggregation)
+ [(sql.qp/->honeysql driver aggregation) direction]))
+ [(sql.qp/->honeysql driver expr) direction]))
+
+(defmethod sql.qp/->honeysql [:sql-mbql5 :value]
+ [driver [op {:keys [base-type effective-type]} value]]
+ ;; We need to rename just these keys for `sql.qp/->honeysql [:sql :value]`
+ ((get-method sql.qp/->honeysql [:sql op]) driver [op value {:base_type base-type :effective_type effective-type}]))
+
+(defmethod sql.qp/->honeysql [:sql-mbql5 :case]
+ [driver [op _opts cases opts-or-default]]
+ ;; Handle the default options coming in as a map from `mbql-clause`, or as a single value from ->mbql5
+ ((get-method sql.qp/->honeysql [:sql op]) driver [op cases (if (map? opts-or-default)
+ opts-or-default
+ {:default opts-or-default})]))
+
+;; For clauses that DO NOT have their opts propagated
+(doseq [op [:!= :* :+ :- :/ :< :<= := :> :>= :abs :absolute-datetime :aggregation-options :and :asc :avg :between
+ ::sql.qp/cast ::sql.qp/cast-to-text :ceil :coalesce :concat :count :count-where :cum-count :cum-sum :date
+ :datetime-add :datetime-diff :datetime-subtract :desc :distinct :distinct-where :exp
+ ::sql.qp/expression-literal-text-value :float :floor :integer :length :log :lower :ltrim :max :median :min
+ ::sql.qp/nfc-path :not :now :or :percentile :power :relative-datetime :replace :round :rtrim :share :sqrt
+ :stddev :substring :sum :sum-where :temporal-extract :text :time :today :trim :upper :var]]
+ (defmethod sql.qp/->honeysql [:sql-mbql5 op]
+ [driver [op _opts & args]]
+ ((get-method sql.qp/->honeysql [:sql op]) driver (into [op] args))))
+
+;; For clauses that DO have their opts propagated
+(doseq [op [:expression :field :datetime :contains :starts-with :ends-with]]
+ (defmethod sql.qp/->honeysql [:sql-mbql5 op]
+ [driver [op opts & args]]
+ ((get-method sql.qp/->honeysql [:sql op]) driver (cond-> (into [op] args) opts (conj opts)))))
+
+(defmethod sql.qp/mbql-clause-with-opts :sql-mbql5
+ [_driver tag opts & args]
+ (into [tag (lib.schema.common/normalize-options-map opts)] args))
+
+(defmethod sql.qp/expression-by-name :sql-mbql5
+ [_driver inner-query expression-name]
+ (m/find-first #(= expression-name (lib.util/expression-name %)) (:expressions inner-query)))
+
+(defmethod sql.qp/aggregation-name :sql-mbql5
+ [_driver inner-query ag-clause]
+ (or (::add/desired-alias (lib/options ag-clause))
+ (:name (lib/options ag-clause))
+ (lib/column-name inner-query ag-clause)))
+
+(defmethod sql.qp/clause-value-idx :sql-mbql5 [_driver] 2)
+
+(defmethod sql.qp/breakout-options-index :sql-mbql5 [_driver] 1)
+
+(mu/defmethod sql.params.substitution/field->clause :sql-mbql5 :- :mbql.clause/field
+ [driver field other-opts]
+ (sql.qp/mbql-clause-with-opts driver :field
+ (merge {:base-type (:base-type field)
+ driver-api/qp.add.source-table (:table-id field)
+ ::sql.params.substitution/compiling-field-filter? true}
+ other-opts)
+ (:id field)))
+
+(defmethod sql.params.substitution/to-clause :sql-mbql5 [_driver param] (qp.params.ops/to-clause param))
+
+(defmethod sql.params.substitution/desugar-filter-clause :sql-mbql5 [_driver filter-clause] (lib/desugar-filter-clause filter-clause))
+
+(defmethod sql.params.substitution/wrap-value-literals-in-mbql :sql-mbql5
+ [_driver mbql]
+ #_{:clj-kondo/ignore [:deprecated-var]}
+ (driver-api/wrap-value-literals-in-mbql5 mbql))
+
+(defmethod sql.params.substitution/date-string->filter :sql-mbql5 [_driver date-string field] (qp.params.dates/date-string->filter date-string field))
diff --git a/src/metabase/driver_api/core.clj b/src/metabase/driver_api/core.clj
index a736e4e2987..2e6e4b4adf3 100644
--- a/src/metabase/driver_api/core.clj
+++ b/src/metabase/driver_api/core.clj
@@ -167,8 +167,8 @@
qp.util.transformations.nest-breakouts/nest-breakouts-in-stages-with-window-aggregation
qp.util/default-query->remark
qp.util/query->remark
- qp.wrap-value-literals/unwrap-value-literal
qp.wrap-value-literals/wrap-value-literals-in-mbql
+ qp.wrap-value-literals/wrap-value-literals-in-mbql5
qp.writeback/execute-write-sql!
qp/process-query
secrets/clean-secret-properties-from-details
diff --git a/src/metabase/lib/convert.cljc b/src/metabase/lib/convert.cljc
index 0460c558d24..c11a1eac55a 100644
--- a/src/metabase/lib/convert.cljc
+++ b/src/metabase/lib/convert.cljc
@@ -445,7 +445,7 @@
(defmethod ->mbql5 ::string-comparison
[[tag opts & args :as clause]]
- (if (> (count args) 2)
+ (if (or (> (count args) 2) (map? opts))
;; Multi-arg, MBQL 5 style: [tag {opts...} x y z ...]
(lib.options/ensure-uuid (into [tag opts] (map ->mbql5 args)))
;; Two-arg, legacy style: [tag x y] or [tag x y opts].
diff --git a/src/metabase/lib/schema/mbql_clause.cljc b/src/metabase/lib/schema/mbql_clause.cljc
index c78a5472050..d48d53c8bba 100644
--- a/src/metabase/lib/schema/mbql_clause.cljc
+++ b/src/metabase/lib/schema/mbql_clause.cljc
@@ -96,7 +96,7 @@
(when (sequential? x)
(if ((some-fn map? nil?) (second x))
x
- (into [(first x) {}] (rest x)))))
+ (into [(first x) {:lib/uuid (str (random-uuid))}] (rest x)))))
;;; TODO: Support options more nicely - these don't allow for overriding the options, but we have a few cases where that
;;; is necessary. See for example the inclusion of `string-filter-options` in [[metabase.lib.filter]].
diff --git a/src/metabase/query_processor/middleware/wrap_value_literals.clj b/src/metabase/query_processor/middleware/wrap_value_literals.clj
index 3f1af6ba93d..8da92e2626e 100644
--- a/src/metabase/query_processor/middleware/wrap_value_literals.clj
+++ b/src/metabase/query_processor/middleware/wrap_value_literals.clj
@@ -277,15 +277,6 @@
;;; Tangentially-related nonsense not used by the middleware
;;;
-;;; TODO (Cam 8/22/25) FIXME: This is used in exactly one place: the SQL QP... so why does it live in a QP middleware
-;;; namespace? Nobody knows.
-(defn unwrap-value-literal
- "Extract value literal from `:value` form or returns form as is if not a `:value` form."
- [maybe-value-form]
- (match/match-one maybe-value-form
- [:value x & _] x
- _ maybe-value-form))
-
(defn- type-info-no-query
"This is like [[type-info*]] but specifically for supporting the legacy/deprecated [[wrap-value-literals-in-mbql]]
function."
@@ -303,7 +294,7 @@
{:base-type expr-type
:effective-type expr-type})))
-(mu/defn wrap-value-literals-in-mbql :- [:cat :keyword [:* :any]]
+(mu/defn wrap-value-literals-in-mbql5 :- [:cat :keyword [:* :any]]
"Given a normalized legacy MBQL query (important to desugar forms like `[:does-not-contain ...]` -> `[:not [:contains
...]]`), walks over the clause and annotates literals with type information.
@@ -329,5 +320,12 @@
[$mbql]
(fn [clause]
(wrap-value-literals-in-clause nil nil clause)))
- first)))
- lib/->legacy-MBQL))
+ first)))))
+
+(mu/defn wrap-value-literals-in-mbql :- [:cat :keyword [:* :any]]
+ "Wrapper around `wrap-value-literals-in-mbql5` that converts the clause back to legacy MBQL.
+ DEPRECATED: This is for legacy compatibility and should not be used in new code."
+ {:deprecated "0.57.0"}
+ [mbql :- [:cat :keyword [:* :any]]]
+ #_{:clj-kondo/ignore [:deprecated-var]}
+ (lib/->legacy-MBQL (wrap-value-literals-in-mbql5 mbql)))
diff --git a/src/metabase/util/experiment.cljc b/src/metabase/util/experiment.cljc
index 0f239704485..1f58f4c6c2a 100644
--- a/src/metabase/util/experiment.cljc
+++ b/src/metabase/util/experiment.cljc
@@ -35,9 +35,8 @@
[f]
(reset! experiments-enabled-fn f))
-(defonce ^{:private true
- :doc "Atom holding the default report function. Set at application startup via [[set-default-report-fn!]].
- Nil means no reporting. Can be overridden per-experiment via :report-fn in the opts map."}
+(defonce ^{:doc "Atom holding the default report function. Set at application startup via [[set-default-report-fn!]].
+ Nil means no reporting. Can be overridden per-experiment via :report-fn in the opts map."}
default-report-fn (atom nil))
(defn set-default-report-fn!
diff --git a/src/metabase/util/honey_sql_2.clj b/src/metabase/util/honey_sql_2.clj
index 48c5aee1239..461f2b2a9b3 100644
--- a/src/metabase/util/honey_sql_2.clj
+++ b/src/metabase/util/honey_sql_2.clj
@@ -436,7 +436,7 @@
"HoneySQL form that should be used to get the current `datetime` (or equivalent), e.g. `:%now`."
[db-type]
(case db-type
- :h2 (with-database-type-info :%now "timestamp")
+ (:h2 :h2-mbql5) (with-database-type-info :%now "timestamp")
:mysql (with-database-type-info [:now [:inline 6]] "timestamp")
:postgres (with-database-type-info :%now "timestamptz")))
@@ -525,6 +525,10 @@
:else
(dateadd-h2 unit amount hsql-form)))
+(defmethod add-interval-honeysql-form :h2-mbql5
+ [db-type hsql-form amount unit]
+ ((get-method add-interval-honeysql-form :h2) db-type hsql-form amount unit))
+
(defmethod add-interval-honeysql-form :default
[db-type hsql-form amount unit]
(throw (ex-info (clojure.core/format (str "metabase.util.honey-sql-2/add-interval-honeysql-form not implemented for db-type %s. "
diff --git a/test/metabase/driver/h2_test.clj b/test/metabase/driver/h2_test.clj
index 0bb540db63f..d5e4d18b86c 100644
--- a/test/metabase/driver/h2_test.clj
+++ b/test/metabase/driver/h2_test.clj
@@ -21,11 +21,34 @@
[metabase.query-processor.preprocess :as qp.preprocess]
[metabase.query-processor.test :as qp]
[metabase.test :as mt]
+ [metabase.test.data.datasets :as mtd]
+ [metabase.test.data.env :as tx.env]
+ [metabase.test.data.interface :as tx]
[metabase.util.honey-sql-2 :as h2x]
[toucan2.core :as t2]))
(set! *warn-on-reflection* true)
+;; temporary hack to run all tests against both the old :h2 and new mbql5 :h2
+;; remove this once :h2 does mbql5 by default!
+(defn- test-driver [driver thunk]
+ (when (contains? (tx.env/test-drivers) driver)
+ (testing (str "\n" driver "\n")
+ (driver/with-driver (tx/the-driver-with-test-extensions driver)
+ (thunk))
+ ;; the above is the original definition of test-driver, but we add in
+ ;; this clause to avoid having to rewrite all the tests below twice:
+ (when (= driver :h2)
+ (driver/with-driver (tx/the-driver-with-test-extensions :h2-mbql5)
+ (thunk))))))
+
+(use-fixtures :each (fn [f]
+ ;; NB: because of test parallelism, this *will* affect other non-h2
+ ;; tests, but the check above in the test-driver function will
+ ;; prevent it from actually doing anything different in those tests.
+ (with-redefs [mtd/-test-driver test-driver]
+ (f))))
+
(deftest ^:parallel parse-connection-string-test
(testing "Check that the functions for exploding a connection string's options work as expected"
(is (= ["file:my-file" {"OPTION_1" "TRUE", "OPTION_2" "100", "LOOK_I_INCLUDED_AN_EXTRA_SEMICOLON" "NICE_TRY"}]
diff --git a/test/metabase/driver/sql/query_processor_test.clj b/test/metabase/driver/sql/query_processor_test.clj
index db9b7ffe998..cb7359185b0 100644
--- a/test/metabase/driver/sql/query_processor_test.clj
+++ b/test/metabase/driver/sql/query_processor_test.clj
@@ -9,6 +9,7 @@
[metabase.driver.sql.query-processor :as sql.qp]
[metabase.driver.sql.query-processor-test-util :as sql.qp-test-util]
[metabase.driver.sql.query-processor.deprecated]
+ [metabase.lib.convert :as lib.convert]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.lib.options :as lib.options]
@@ -17,6 +18,7 @@
[metabase.lib.test-util.macros :as lib.tu.macros]
[metabase.lib.test-util.notebook-helpers :as lib.tu.notebook]
[metabase.lib.test-util.places-cam-likes-metadata-provider :as lib.tu.places-cam-likes-metadata-provider]
+ [metabase.lib.util :as lib.util]
[metabase.query-processor.compile :as qp.compile]
[metabase.query-processor.middleware.limit :as limit]
[metabase.query-processor.preprocess :as qp.preprocess]
@@ -221,12 +223,18 @@
(testing "params from source queries should get passed in to the top-level. Semicolons should be removed"
(is (= {:query "SELECT \"__mb_source\".* FROM (SELECT * FROM some_table WHERE name = ?) AS \"__mb_source\" WHERE (\"__mb_source\".\"name\" <> ?) OR (\"__mb_source\".\"name\" IS NULL)"
:params ["Cam" "Lucky Pigeon"]}
- (sql.qp/mbql->native
- :h2
- (lib.tu.macros/mbql-query venues
- {:source-query {:native "SELECT * FROM some_table WHERE name = ?;", :params ["Cam"]}
- :source-metadata [{:name "name", :display_name "Name", :base_type :type/Integer}]
- :filter [:!= *name/Integer "Lucky Pigeon"]}))))))))
+ (->> {:source-query {:native "SELECT * FROM some_table WHERE name = ?;", :params ["Cam"]}
+ :source-metadata [{:name "name", :display_name "Name", :base_type :type/Integer}]
+ :filter [:!= *name/Integer "Lucky Pigeon"]}
+ (lib.tu.macros/mbql-query venues)
+ (lib.convert/->mbql5)
+ (lib/query meta/metadata-provider)
+ (sql.qp/mbql->native :h2))))))))
+
+(defn- legacy-join->honeysql [driver join]
+ (sql.qp/join->honeysql driver (if (isa? driver/hierarchy driver :sql-mbql5)
+ (lib.convert/->mbql5 (#'lib.util/join->pipeline join))
+ join)))
(deftest ^:parallel joins-against-native-queries-test
(testing "Joins against native SQL queries should get converted appropriately! make sure correct HoneySQL is generated"
@@ -237,33 +245,31 @@
[:=
(h2x/with-database-type-info (h2x/identifier :field "PUBLIC" "CHECKINS" "VENUE_ID") "integer")
(h2x/identifier :field "card" "id")]]
- (sql.qp/join->honeysql :h2
- (lib.tu.macros/$ids checkins
+ (legacy-join->honeysql :h2
+ {:source-query {:native "SELECT * FROM VENUES;", :params []}
+ :alias "card"
+ :strategy :left-join
+ :condition [:=
+ [:field (meta/id :checkins :venue-id) {::add/source-table (meta/id :checkins)
+ ::add/source-alias "VENUE_ID"}]
+ [:field "id" {:join-alias "card"
+ :base-type :type/Integer
+ ::add/source-table "card"
+ ::add/source-alias "id"}]]})))))))
+
+(defn- compile-join [driver]
+ (driver/with-driver driver
+ (qp.store/with-metadata-provider meta/metadata-provider
+ (let [join (legacy-join->honeysql driver
{:source-query {:native "SELECT * FROM VENUES;", :params []}
:alias "card"
:strategy :left-join
:condition [:=
- [:field %venue-id {::add/source-table $$checkins
- ::add/source-alias "VENUE_ID"}]
- [:field "id" {:join-alias "card"
- :base-type :type/Integer
+ [:field (meta/id :checkins :id) {::add/source-table (meta/id :checkins)
+ ::add/source-alias "VENUE_ID"}]
+ [:field "id" {:base-type :type/Text
::add/source-table "card"
- ::add/source-alias "id"}]]}))))))))
-
-(defn- compile-join [driver]
- (driver/with-driver driver
- (qp.store/with-metadata-provider meta/metadata-provider
- (let [join (sql.qp/join->honeysql
- driver
- {:source-query {:native "SELECT * FROM VENUES;", :params []}
- :alias "card"
- :strategy :left-join
- :condition [:=
- [:field (meta/id :checkins :id) {::add/source-table (meta/id :checkins)
- ::add/source-alias "VENUE_ID"}]
- [:field "id" {:base-type :type/Text
- ::add/source-table "card"
- ::add/source-alias "id"}]]})]
+ ::add/source-alias "id"}]]})]
(sql.qp/format-honeysql driver {:join join})))))
;;; Ok to hardcode driver names here because it's for general HoneySQL compilation behavior and not something that needs
diff --git a/test/metabase/driver/sql_jdbc_test.clj b/test/metabase/driver/sql_jdbc_test.clj
index 93852bdc357..555405658d7 100644
--- a/test/metabase/driver/sql_jdbc_test.clj
+++ b/test/metabase/driver/sql_jdbc_test.clj
@@ -233,48 +233,52 @@
[[uuid]] (lib/ends-with col (str uuid))
[[uuid]] (lib/contains col (str uuid))
- ;; Test partial uuid values
+ ;; Test partial uuid values
[[uuid]] (lib/contains col (subs (str uuid) 0 1))
[[uuid]] (lib/starts-with col (subs (str uuid) 0 1))
[[uuid]] (lib/ends-with col (subs (str uuid) (dec (count (str uuid)))))
- ;; Cannot match a uuid, but should not blow up
+ ;; Cannot match a uuid, but should not blow up
[[uuid]] (lib/!= col "q")
[] (lib/= col "q")
[] (lib/starts-with col "q")
[] (lib/ends-with col "q")
[] (lib/contains col "q")
- ;; empty/null handling
+ ;; empty/null handling
[] (lib/is-empty col)
[[uuid]] (lib/not-empty col)
[] (lib/is-null col)
[[uuid]] (lib/not-null col)
- ;; nil value handling
+ ;; nil value handling
[[uuid]] (lib/!= col nil)
[] (lib/= col nil))
- (testing ":= uses indexable query"
- (is (=? [:= [:metabase.util.honey-sql-2/identifier :field [(second (lib/->legacy-MBQL (lib/ref col)))]]
- (some-fn #(= uuid %)
- #(= [:metabase.util.honey-sql-2/typed
- [:cast (str uuid) [:raw "uuid"]]
- {:database-type "uuid"}]
- %))]
- (sql.qp/->honeysql
- driver/*driver*
- [:=
- (lib/->legacy-MBQL (lib/ref col))
- [:value (str uuid) {:base_type :type/UUID}]])))
- (is (=? [:= [:metabase.util.honey-sql-2/identifier :field [(second (lib/->legacy-MBQL (lib/ref col)))]]
- (some-fn #(= uuid %)
- #(= [:metabase.util.honey-sql-2/typed
- [:cast (str uuid) [:raw "uuid"]]
- {:database-type "uuid"}]
- %))]
- (sql.qp/->honeysql
- driver/*driver*
- [:= (lib/->legacy-MBQL (lib/ref col)) uuid])))))))
+ (let [mbql5-driver? (isa? driver/hierarchy driver/*driver* :sql-mbql5)
+ field (get (lib/ref col) 2)
+ col-ref (cond-> (lib/ref col)
+ (not mbql5-driver?) lib/->legacy-MBQL)]
+ (testing ":= uses indexable query"
+ (is (=? [:= [:metabase.util.honey-sql-2/identifier :field [field]]
+ (some-fn #(= uuid %)
+ #(= [:metabase.util.honey-sql-2/typed
+ [:cast (str uuid) [:raw "uuid"]]
+ {:database-type "uuid"}]
+ %))]
+ (sql.qp/->honeysql
+ driver/*driver*
+ (sql.qp/mbql-clause driver/*driver*
+ := col-ref
+ (sql.qp/mbql-clause-with-opts driver/*driver* :value {:base_type :type/UUID} (str uuid))))))
+ (is (=? [:= [:metabase.util.honey-sql-2/identifier :field [field]]
+ (some-fn #(= uuid %)
+ #(= [:metabase.util.honey-sql-2/typed
+ [:cast (str uuid) [:raw "uuid"]]
+ {:database-type "uuid"}]
+ %))]
+ (sql.qp/->honeysql
+ driver/*driver*
+ (sql.qp/mbql-clause driver/*driver* := col-ref uuid)))))))))
(deftest query-canceled-test?
(testing "walks a chain of exceptions"
diff --git a/test/metabase/query_processor/date_bucketing_test.clj b/test/metabase/query_processor/date_bucketing_test.clj
index 762c404fe98..2503d7f456c 100644
--- a/test/metabase/query_processor/date_bucketing_test.clj
+++ b/test/metabase/query_processor/date_bucketing_test.clj
@@ -1810,7 +1810,7 @@
#t "2022-03-31T00:00:00"
#t "2022-03-31T00:00:00-00:00"]]
(testing (format "%d %s ^%s %s" n unit (.getCanonicalName (class t)) (pr-str t))
- (let [march-31 (sql.qp/->honeysql driver/*driver* [:absolute-datetime t :day])
+ (let [march-31 (sql.qp/->honeysql driver/*driver* (sql.qp/mbql-clause driver/*driver* :absolute-datetime t :day))
june-31 (sql.qp/add-interval-honeysql-form driver/*driver* march-31 n unit)
checkins (mt/with-metadata-provider (mt/id)
(sql.qp/->honeysql driver/*driver* (lib.metadata/table (qp.store/metadata-provider) (mt/id :checkins))))
diff --git a/test/metabase/query_processor/expression_aggregations_test.clj b/test/metabase/query_processor/expression_aggregations_test.clj
index 48d2abdbeee..41265a92615 100644
--- a/test/metabase/query_processor/expression_aggregations_test.clj
+++ b/test/metabase/query_processor/expression_aggregations_test.clj
@@ -4,6 +4,7 @@
[clojure.test :refer :all]
[metabase.driver :as driver]
[metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
[metabase.lib.test-util :as lib.tu]
[metabase.query-processor.test :as qp]
[metabase.query-processor.test-util :as qp.test-util]
@@ -395,3 +396,26 @@
mt/rows
(map second)
(map sort))))))))
+
+(deftest ^:parallel aggregation-function-inside-expression-in-aggregation-test
+ (testing "should allow using aggregation functions inside expressions in aggregation (metabase#52611)"
+ (mt/test-drivers (mt/normal-driver-select {:+features [:expression-aggregations :binning]})
+ (let [mp (mt/metadata-provider)
+ orders (lib.metadata/table mp (mt/id :orders))
+ total (lib.metadata/field mp (mt/id :orders :total))
+ subtotal (lib.metadata/field mp (mt/id :orders :subtotal))
+ query (-> (lib/query mp orders)
+ (lib/aggregate
+ (lib/case [[(lib/> (lib/sum total) 10) (lib/sum total)]]
+ (lib/sum subtotal)))
+ (lib/breakout (lib/with-binning total {:strategy :bin-width :bin-width 20})))]
+ (is (= [[-60 26.96]
+ [0 851.99]
+ [20 71843.32]
+ [40 204530.21]
+ [60 283033.59]
+ [80 233639.15]
+ [100 270513.03]
+ [120 317763.17]
+ [140 128488.72]]
+ (mt/formatted-rows [int 2.0] (qp/process-query query))))))))
diff --git a/test/metabase/test/data/h2_mbql5.clj b/test/metabase/test/data/h2_mbql5.clj
new file mode 100644
index 00000000000..e43e78486b4
--- /dev/null
+++ b/test/metabase/test/data/h2_mbql5.clj
@@ -0,0 +1,2 @@
+(ns metabase.test.data.h2-mbql5
+ "Placeholder; delete me once experimental-mbql5-driver goes away.")