Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Migrate MongoDB to MBQL 5 (#77673)

Cam Saul34863099c0ea
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index fa539cf99ff..e92f2d23137 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -1213,7 +1213,8 @@
metabase.lib.schema.util
metabase.lib.schema.validate
metabase.lib.types.isa
- metabase.lib.util}
+ metabase.lib.util
+ metabase.lib.walk}
:uses #{config
graph
legacy-mbql
diff --git a/.clj-kondo/ratchets.edn b/.clj-kondo/ratchets.edn
index de25c512284..5be40dd0617 100644
--- a/.clj-kondo/ratchets.edn
+++ b/.clj-kondo/ratchets.edn
@@ -7,10 +7,10 @@
{:ignore-counts {:aliased-namespace-symbol 3
:all 49
:case-symbol-test 1
- :clojure-lsp/unused-public-var 10
+ :clojure-lsp/unused-public-var 9
:def-fn 1
:deprecated-namespace 95
- :deprecated-var 79
+ :deprecated-var 84
:discouraged-java-method 3
:discouraged-namespace 80
:discouraged-var 107
diff --git a/docs/developers-guide/driver-changelog.md b/docs/developers-guide/driver-changelog.md
index 13e494c0b65..c88b5edc4a4 100644
--- a/docs/developers-guide/driver-changelog.md
+++ b/docs/developers-guide/driver-changelog.md
@@ -58,6 +58,12 @@ title: Driver interface changelog
- `metabase.driver/compile-create-index` `[driver schema table structured]` -- compiles a `:standalone` index into
the DDL statement(s) that create it.
+- `metabase.driver-api.core/aggregation-name` is now deprecated, as it operates on legacy MBQL; use
+ `metabase.driver-api.core/mbql-5-aggregation-name` going forward.
+
+- The method signature for `metabase.driver/substitute-native-parameters-in-stage-method`, introduced in 63, has
+ changed; please update your implementations accordingly.
+
## Metabase 0.63.0
- `metabase.driver/refresh-table-stats!` `[driver database schema table transform-type]` -- refreshes table
diff --git a/modules/drivers/mongo/src/metabase/driver/mongo.clj b/modules/drivers/mongo/src/metabase/driver/mongo.clj
index 5c1cce02b54..c4cf34ae107 100644
--- a/modules/drivers/mongo/src/metabase/driver/mongo.clj
+++ b/modules/drivers/mongo/src/metabase/driver/mongo.clj
@@ -18,10 +18,13 @@
[metabase.driver.mongo.util :as mongo.util]
[metabase.driver.settings :as driver.settings]
[metabase.driver.util :as driver.u]
+ [metabase.lib.schema :as lib.schema]
+ [metabase.lib.schema.id :as lib.schema.id]
[metabase.util :as u]
[metabase.util.date-2 :as u.date]
[metabase.util.json :as json]
[metabase.util.log :as log]
+ [metabase.util.malli :as mu]
[metabase.util.performance :refer [some mapv empty? get-in]]
[taoensso.nippy :as nippy])
(:import
@@ -183,14 +186,16 @@
:semantic-version sanitized-version-array})))
(defmethod driver/describe-database* :mongo
- [_ database]
+ [_driver database]
(mongo.connection/with-mongo-database [^MongoDatabase db database]
- {:tables (set (for [collection (mongo.util/list-collection-names db)
- :when (not (contains? #{"system.indexes" "system.views"} collection))]
- {:schema nil, :name collection}))}))
+ {:tables (into #{}
+ (comp (remove #{"system.indexes" "system.views"})
+ (map (fn [collection]
+ {:schema nil, :name collection})))
+ (mongo.util/list-collection-names db))}))
(defmethod driver/describe-table-indexes :mongo
- [_ database table]
+ [_driver database table]
(mongo.connection/with-mongo-database [^MongoDatabase db database]
(let [collection (mongo.util/collection db (:name table))]
(try
@@ -210,18 +215,21 @@
:value %}))
set)
(catch com.mongodb.MongoCommandException e
- ;; com.mongodb.MongoCommandException: Command failed with error 166 (CommandNotSupportedOnView): 'Namespace test-data.orders_m is a view, not a collection' on server bee:27017. The full response is {"ok": 0.0, "errmsg": "Namespace test-data.orders_m is a view, not a collection", "code": 166, "codeName": "CommandNotSupportedOnView"}
+ ;; com.mongodb.MongoCommandException: Command failed with error 166 (CommandNotSupportedOnView): 'Namespace
+ ;; test-data.orders_m is a view, not a collection' on server bee:27017. The full response is {"ok":
+ ;; 0.0, "errmsg": "Namespace test-data.orders_m is a view, not a collection", "code":
+ ;; 166, "codeName": "CommandNotSupportedOnView"}
(if (= (.getErrorCode e) 166)
#{}
(throw e)))))))
;; describe-table impl start
-(def describe-table-query-depth
- "The depth of nested objects that [[describe-table-query]] will execute to. If set to 0, the query will only return the
- fields at the root level of the document. If set to K, the query will return fields at K levels of nesting beyond that.
- Setting its value involves a trade-off: the lower it is, the faster describe-table-query executes, but the more queries we might
- have to execute."
+(def ^:private describe-table-query-depth
+ "The depth of nested objects that [[describe-table-query]] will execute to. If set to 0, the query will only return
+ the fields at the root level of the document. If set to K, the query will return fields at K levels of nesting
+ beyond that. Setting its value involves a trade-off: the lower it is, the faster describe-table-query executes, but
+ the more queries we might have to execute."
;; Cal 2024-09-15: I chose 100 as the limit because it's a pretty safe bet it won't be exceeded (the documents we've
;; seen on cloud are all <20 levels deep)
;; Case 2024-10-04: Sharded clusters seem to run into exponentially more work the bigger this is. Over 20 and this
@@ -232,9 +240,10 @@
;; > If people have problems with that, I think we can make it configurable.
7)
-(defn ^:dynamic *sample-stages*
+(mu/defn- ^:dynamic *sample-stages*
"Stages to get sample of a collection in [[describe-table-pipeline]]. Dynamic for testing purposes."
- [collection-name sample-size]
+ [collection-name :- :string
+ sample-size :- pos-int?]
(let [start-n (quot sample-size 2)
end-n (- sample-size start-n)]
[{"$sort" {"_id" 1}}
@@ -470,7 +479,7 @@
([acc row]
(conj! acc (zipmap [:path :type :indices] row)))))
-(defn- fetch-dbfields
+(mu/defn- fetch-dbfields
"Fetch _dbfields_ from a mongo collection.
Result is vector of maps as
@@ -478,15 +487,18 @@
Each row represents leaf in sampled documents, its type and indices of keys present in the path of mongo of nested
object."
- [database table]
- (let [pipeline (describe-table-pipeline {:collection-name (:name table)
- :sample-size (* table-rows-sample/nested-field-sample-limit 2)
+ [database :- [:map
+ [:id ::lib.schema.id/database]]
+ table :- [:map
+ [:name :string]]]
+ (let [pipeline (describe-table-pipeline {:collection-name (:name table)
+ :sample-size (* table-rows-sample/nested-field-sample-limit 2)
:document-sample-depth describe-table-query-depth
- :leaf-limit (driver.settings/sync-leaf-fields-limit)})
- query {:database (:id database)
- :type "native"
- :native {:collection (:name table)
- :query (json/encode pipeline)}}]
+ :leaf-limit (driver.settings/sync-leaf-fields-limit)})
+ query {:database (:id database)
+ :type "native"
+ :native {:collection (:name table)
+ :query (json/encode pipeline)}}]
(driver-api/process-query query fetch-dbfields-rff)))
(defmethod driver/describe-table :mongo
@@ -499,35 +511,35 @@
;; describe-table impl end
-(doseq [[feature supported?] {:basic-aggregations true
- :expression-aggregations true
- :expression-literals true
- :inner-join true
- :left-join true
- :nested-fields true
- :native-parameter-card-reference false
- :native-parameters true
- :nested-queries true
- :set-timezone true
- :standard-deviation-aggregations true
- :test/create-table-without-data false
- :rename true
- :test/jvm-timezone-setting false
- :identifiers-with-spaces true
- :saved-question-sandboxing false
- :expressions/date true
- :expressions/text true
- :expressions/datetime true
- :expressions/today true
- ;; Index sync is turned off across the application as it is not used ATM.
- :index-info false
- :python-transforms true
- :transforms/python true
- :database-routing true}]
+(doseq [[feature supported?] {:basic-aggregations true
+ :database-routing true
+ :expression-aggregations true
+ :expression-literals true
+ :expressions/date true
+ :expressions/datetime true
+ :expressions/text true
+ :expressions/today true
+ :identifiers-with-spaces true
+ :index-info false ; Index sync is turned off across the application as it is not used ATM.
+ :inner-join true
+ :left-join true
+ :metadata/key-constraints false
+ :native-parameter-card-reference false
+ :native-parameters true
+ :native-requires-specified-collection true
+ :nested-fields true
+ :nested-queries true
+ :python-transforms true
+ :rename true
+ :saved-question-sandboxing false
+ :schemas false
+ :set-timezone true
+ :standard-deviation-aggregations true
+ :test/create-table-without-data false
+ :test/jvm-timezone-setting false
+ :transforms/python true}]
(defmethod driver/database-supports? [:mongo feature] [_driver _feature _db] supported?))
-(defmethod driver/database-supports? [:mongo :schemas] [_driver _feat _db] false)
-
(defn- dbms-version [database]
;; avoid trying `:dbms_version` if `:dbms-version` is present but `nil`; this will cause snake-hating-map warnings
(when-let [k (some #(when (contains? database %)
@@ -567,16 +579,9 @@
:semantic-version
(driver.u/semantic-version-gte [4 2])))
-(defmethod driver/database-supports? [:mongo :native-requires-specified-collection]
- [_driver _feature _db]
- true)
-
-(defmethod driver/database-supports? [:mongo :metadata/key-constraints]
- [_driver _feature _db]
- false)
-
-(defmethod driver/mbql->native :mongo
- [_ query]
+(mu/defmethod driver/mbql->native :mongo :- :metabase.query-processor.compile/compiled
+ [_driver :- :keyword
+ query :- ::lib.schema/query]
(mongo.qp/mbql->native query))
(defmethod driver/execute-reducible-query :mongo
@@ -586,8 +591,8 @@
(mongo.execute/execute-reducible-query query respond)))
(defmethod driver/substitute-native-parameters-in-stage-method :mongo
- [driver metadata-providerable stage]
- (mongo.params/substitute-native-parameters driver metadata-providerable stage))
+ [driver query stage-number]
+ (mongo.params/substitute-native-parameters driver query stage-number))
(defmethod driver/db-start-of-week :mongo
[_]
diff --git a/modules/drivers/mongo/src/metabase/driver/mongo/execute.clj b/modules/drivers/mongo/src/metabase/driver/mongo/execute.clj
index 9d5cd6038c6..d334d097dab 100644
--- a/modules/drivers/mongo/src/metabase/driver/mongo/execute.clj
+++ b/modules/drivers/mongo/src/metabase/driver/mongo/execute.clj
@@ -34,7 +34,7 @@
"Returns a map of column name in result row -> unescaped column name to return in metadata e.g.
{\"date_field~~~month\" \"date_field\"}"
- [projections :- mongo.qp/Projections]
+ [projections :- ::mongo.qp/projections]
(into
{}
(for [k projections
@@ -42,10 +42,11 @@
:when (not (= k unescaped))]
[k unescaped])))
-(defn check-columns
+(mu/defn- check-columns
"Make sure there are no columns coming back from `results` that we weren't expecting. If there are, we did something
wrong here and the query we generated is off."
- [columns first-row-col-names]
+ [columns :- ::mongo.qp/projections
+ first-row-col-names] ; first row col names seems to be `java.util.LinkedHashMap$LinkedKeySet` so don't try to Malli it
{:pre [(every? string? columns) (every? string? first-row-col-names)]}
(when (seq first-row-col-names)
(let [expected-cols (set (for [col-name columns]
@@ -82,12 +83,12 @@
[:unescaped [:maybe [:sequential :string]]]]
"Return column names we can expect in each `:row` of the results, and the `:unescaped` versions we should return in
thr query result metadata."
- [{:keys [mbql? projections]} :- :map
+ [{:keys [mbql? projections]} :- ::mongo.qp/compiled-pipeline-with-collection
query
first-row-col-names]
;; some of the columns may or may not come back in every row, because of course with mongo some key can be missing.
;; That's ok, the logic below where we call `(mapv row columns)` will end up adding `nil` results for those columns.
- (if-not (and mbql? projections)
+ (if-not (and mbql? (seq projections))
(let [project-stage (->> query (filter #(contains? % "$project")) last)
projected (keep (fn [[k v]] (when-not (contains? suppressing-values v) k))
(get project-stage "$project"))
diff --git a/modules/drivers/mongo/src/metabase/driver/mongo/parameters.clj b/modules/drivers/mongo/src/metabase/driver/mongo/parameters.clj
index d64638a512c..75f8df97391 100644
--- a/modules/drivers/mongo/src/metabase/driver/mongo/parameters.clj
+++ b/modules/drivers/mongo/src/metabase/driver/mongo/parameters.clj
@@ -62,22 +62,25 @@
(pr-str x)))
(mu/defn- field->name
- [field :- driver-api/schema.metadata.column
- alias]
- (let [name (if (str/blank? alias)
- (mongo.qp/field->name field ".")
- alias)]
+ [metadata-providerable :- ::lib.schema.metadata/metadata-providerable
+ field :- driver-api/schema.metadata.column
+ column-alias :- [:maybe :string]]
+ (let [name (if (str/blank? column-alias)
+ (mongo.qp/field->name metadata-providerable field ".")
+ column-alias)]
(pr-str name)))
-(defn- substitute-one-field-filter-date-range [{field :field, alias :alias, {value :value} :value}]
+(mu/defn- substitute-one-field-filter-date-range
+ [metadata-providerable :- ::lib.schema.metadata/metadata-providerable
+ {field :field, alias :alias, {value :value} :value}]
(let [{:keys [start end]} (params.dates/date-string->range value {:inclusive-end? false})
start-condition (when start
(format "{%s: {$gte: %s}}"
- (field->name field alias)
+ (field->name metadata-providerable field alias)
(param-value->str field (u.date/parse start ZoneOffset/UTC))))
end-condition (when end
(format "{%s: {$lt: %s}}"
- (field->name field alias)
+ (field->name metadata-providerable field alias)
(param-value->str field (u.date/parse end ZoneOffset/UTC))))]
(if (and start-condition end-condition)
(format "{$and: [%s, %s]}" start-condition end-condition)
@@ -87,28 +90,30 @@
;; Field filter value is either lib/parsed-param-no-value-placeholder (handled in `substitute-param`, a map with `:type` and `:value`, or a
;; sequence of those maps.
(mu/defn- substitute-one-field-filter
- [{:keys [field alias], {param-type :type, value :value} :value, :as field-filter} :- :metabase.lib.parameters.parse.types/field-filter]
+ [metadata-providerable :- ::lib.schema.metadata/metadata-providerable
+ {:keys [field alias], {param-type :type, value :value} :value, :as field-filter} :- :metabase.lib.parameters.parse.types/field-filter]
;; convert relative dates to appropriate date range representations
(cond
(params.dates/not-single-date-type? param-type)
- (substitute-one-field-filter-date-range field-filter)
+ (substitute-one-field-filter-date-range metadata-providerable field-filter)
;; a `date/single` like `2020-01-10`
(and (params.dates/date-type? param-type)
(string? value))
(let [t (u.date/parse value)]
(format "{$and: [%s, %s]}"
- (format "{%s: {$gte: %s}}" (field->name field alias) (param-value->str field t))
- (format "{%s: {$lt: %s}}" (field->name field alias) (param-value->str field (u.date/add t :day 1)))))
+ (format "{%s: {$gte: %s}}" (field->name metadata-providerable field alias) (param-value->str field t))
+ (format "{%s: {$lt: %s}}" (field->name metadata-providerable field alias) (param-value->str field (u.date/add t :day 1)))))
:else
- (format "{%s: %s}" (field->name field alias) (param-value->str field value))))
+ (format "{%s: %s}" (field->name metadata-providerable field alias) (param-value->str field value))))
(mu/defn- substitute-field-filter
- [{field :field, alias :alias, {:keys [value]} :value, :as field-filter} :- :metabase.lib.parameters.parse.types/field-filter]
+ [metadata-providerable :- ::lib.schema.metadata/metadata-providerable
+ {field :field, alias :alias, {:keys [value]} :value, :as field-filter} :- :metabase.lib.parameters.parse.types/field-filter]
(if (sequential? value)
- (format "{%s: %s}" (field->name field alias) (param-value->str field value))
- (substitute-one-field-filter field-filter)))
+ (format "{%s: %s}" (field->name metadata-providerable field alias) (param-value->str field value))
+ (substitute-one-field-filter metadata-providerable field-filter)))
(mu/defn- substitute-native-query-snippet
[[acc missing]
@@ -116,7 +121,9 @@
[(conj acc content) missing])
(mu/defn- substitute-param
- [param->value
+ [query :- ::lib.schema/query
+ stage-number :- :int
+ param->value
[acc missing]
in-optional?
{:keys [k], :as _param} :- :metabase.lib.parameters.parse.types/param]
@@ -131,7 +138,7 @@
(params.ops/operator? (get-in v [:value :type]))
(let [param (:value v)
field-name (if (str/blank? (:alias v))
- (mongo.qp/field->name (:field v) ".")
+ (mongo.qp/field->name query (:field v) ".")
(:alias v))
compiled-clause (-> (assoc param
:target
@@ -152,19 +159,14 @@
;; [:or [:= <field> 1] ...].
lib/desugar-filter-clause
driver-api/wrap-value-literals-in-mbql5
- ;;
- ;; TODO (Cam 2026-05-14) -- remove once we convert the MongoDB driver to use MBQL
- ;; 5 natively
- ;;
- lib/->legacy-MBQL
- mongo.qp/compile-filter
+ (->> (mongo.qp/compile-filter query stage-number))
json/encode)]
[(conj acc compiled-clause) missing])
;; no-value field filters inside optional clauses are ignored and omitted entirely
(and no-value? in-optional?) [acc (conj missing k)]
;; otherwise replace it with a {} which is the $match equivalent of 1 = 1, i.e. always true
no-value? [(conj acc "{}") missing]
- :else [(conj acc (substitute-field-filter v))
+ :else [(conj acc (substitute-field-filter query v))
missing]))
(lib/parsed-referenced-query-snippet-param? v)
@@ -183,17 +185,23 @@
(declare substitute*)
(mu/defn- substitute-optional
- [param->value
+ [query :- ::lib.schema/query
+ stage-number :- :int
+ param->value
[acc missing]
{subclauses :args} :- :metabase.lib.parameters.parse.types/optional]
- (let [[opt-acc opt-missing] (substitute* param->value subclauses true)]
+ (let [[opt-acc opt-missing] (substitute* query stage-number param->value subclauses true)]
(if (seq opt-missing)
[acc missing]
[(into acc opt-acc) missing])))
-(defn- substitute*
+(mu/defn- substitute*
"Returns a sequence of `[[replaced...] missing-parameters]`."
- [param->value xs in-optional?]
+ [query :- ::lib.schema/query
+ stage-number :- :int
+ param->value
+ xs
+ in-optional?]
(reduce
(fn [[acc missing] x]
(cond
@@ -201,10 +209,10 @@
[(conj acc x) missing]
(lib/parsed-param? x)
- (substitute-param param->value [acc missing] in-optional? x)
+ (substitute-param query stage-number param->value [acc missing] in-optional? x)
(lib/parsed-optional-param? x)
- (substitute-optional param->value [acc missing] x)
+ (substitute-optional query stage-number param->value [acc missing] x)
:else
(throw (ex-info (tru "Don''t know how to substitute {0} {1}" (.getName (class x)) (pr-str x))
@@ -213,24 +221,33 @@
xs))
(mu/defn- substitute
- [param->value :- [:maybe [:map-of :string :any]]
+ [query :- ::lib.schema/query
+ stage-number :- :int
+ param->value :- [:maybe [:map-of :string :any]]
xs]
- (let [[replaced missing] (substitute* param->value xs false)]
+ (let [[replaced missing] (substitute* query stage-number param->value xs false)]
(when (seq missing)
(throw (ex-info (tru "Cannot run query: missing required parameters: {0}" (set missing))
{:type driver-api/qp.error-type.invalid-query})))
(when (seq replaced)
(str/join replaced))))
-(defn- parse-and-substitute [param->value x]
+(mu/defn- parse-and-substitute
+ [query :- ::lib.schema/query
+ stage-number :- :int
+ param->value
+ x]
(if-not (string? x)
x
- (substitute param->value (lib/parse-parameters x false))))
+ (substitute query stage-number param->value (lib/parse-parameters x false))))
-(mu/defn substitute-native-parameters :- ::lib.schema/stage.native
+(mu/defn substitute-native-parameters :- ::lib.schema/query
"Implementation of [[metabase.driver/substitute-native-parameters]] for MongoDB."
- [_driver :- :keyword
- metadata-providerable :- ::lib.schema.metadata/metadata-providerable
- stage :- ::lib.schema/stage.native]
- (let [param->value (params.values/stage->params-map metadata-providerable stage)]
- (update stage :native (partial perf/postwalk (partial parse-and-substitute param->value)))))
+ [_driver :- :keyword
+ query :- ::lib.schema/query
+ stage-number :- :int]
+ (let [param->value (params.values/stage->params-map query (lib/query-stage query stage-number))
+ update-stage (fn [native-stage]
+ (perf/postwalk (partial parse-and-substitute query stage-number param->value)
+ native-stage))]
+ (lib/update-query-stage query stage-number update :native update-stage)))
diff --git a/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj b/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj
index 560ca8cd7f0..82d33783dec 100644
--- a/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj
+++ b/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj
@@ -1,7 +1,7 @@
(ns metabase.driver.mongo.query-processor
"Logic for translating MBQL queries into Mongo Aggregation Pipeline queries. See
https://docs.mongodb.com/manual/reference/operator/aggregation-pipeline/ for more details."
- (:refer-clojure :exclude [some mapv select-keys empty?])
+ (:refer-clojure :exclude [some mapv empty? get-in update-keys])
(:require
[clojure.set :as set]
[clojure.string :as str]
@@ -24,13 +24,25 @@
$toBool $toLower $unwind $year]]
[metabase.driver.util :as driver.u]
[metabase.lib.core :as lib]
+ [metabase.lib.equality :as lib.equality]
+ [metabase.lib.schema :as lib.schema]
+ [metabase.lib.schema.common :as lib.schema.common]
+ [metabase.lib.schema.expression :as lib.schema.expression]
+ [metabase.lib.schema.expression.temporal :as lib.schema.expression.temporal]
+ [metabase.lib.schema.id :as lib.schema.id]
+ [metabase.lib.schema.join :as lib.schema.join]
+ [metabase.lib.schema.mbql-clause :as lib.schema.mbql-clause]
+ [metabase.lib.schema.metadata :as lib.schema.metadata]
+ [metabase.lib.schema.temporal-bucketing :as lib.schema.temporal-bucketing]
+ [metabase.lib.walk :as lib.walk]
[metabase.util :as u]
[metabase.util.date-2 :as u.date]
[metabase.util.i18n :refer [tru]]
[metabase.util.log :as log]
[metabase.util.malli :as mu]
+ [metabase.util.malli.registry :as mr]
[metabase.util.match :as match]
- [metabase.util.performance :as perf :refer [some mapv select-keys empty?]])
+ [metabase.util.performance :as perf :refer [empty? get-in mapv some update-keys]])
(:import
(org.bson BsonBinarySubType)
(org.bson.types Binary ObjectId)))
@@ -44,64 +56,182 @@
;; this is just a very limited schema to make sure we're generating valid queries. We should expand it more in the
;; future
-(def ^:private $ProjectStage [:map-of [:= $project] [:map-of driver-api/schema.common.non-blank-string :any]])
-(def ^:private $SortStage [:map-of [:= $sort] [:map-of driver-api/schema.common.non-blank-string [:enum -1 1]]])
-(def ^:private $MatchStage [:map-of [:= $match] [:map-of
- [:and
- [:or driver-api/schema.common.non-blank-string :keyword]
- [:fn
- {:error/message "not a $not condition"}
- (complement #{:$not "$not"})]]
- :any]])
-(def ^:private $GroupStage [:map-of [:= $group] [:map-of driver-api/schema.common.non-blank-string :any]])
-(def ^:private $AddFieldsStage [:map-of [:= $addFields] [:map-of driver-api/schema.common.non-blank-string :any]])
-(def ^:private $LookupStage [:map-of [:= $lookup] [:map-of [:or :keyword :string] :any]])
-(def ^:private $UnwindStage [:map-of [:= $unwind] [:map-of [:or :keyword :string] :any]])
-(def ^:private $LimitStage [:map-of [:= $limit] pos-int?])
-(def ^:private $SkipStage [:map-of [:= $skip] pos-int?])
-(def ^:private $SetWindowFieldsStage [:map-of [:= $setWindowFields] [:map-of driver-api/schema.common.non-blank-string :any]])
-
-(def ^:private Stage
- [:and
- :map
- [:fn
- {:error/message "map with a single key"}
- #(= (count %) 1)]
- [:multi
- {:dispatch (fn [m]
- (first (keys m)))}
- [$project $ProjectStage]
- [$sort $SortStage]
- [$group $GroupStage]
- [$addFields $AddFieldsStage]
- [$lookup $LookupStage]
- [$unwind $UnwindStage]
- [$match $MatchStage]
- [$limit $LimitStage]
- [$skip $SkipStage]
- [$setWindowFields $SetWindowFieldsStage]]])
-
-(def ^:private Pipeline [:sequential Stage])
-
-(def Projections
+(mr/def ::$project-stage
+ [:map-of
+ [:= "$project"]
+ [:map-of ::lib.schema.common/non-blank-string :any]])
+
+(mr/def ::sort-spec
+ [:map-of ::lib.schema.common/non-blank-string [:enum -1 1]])
+
+(mr/def ::$sort-stage
+ [:map-of
+ [:= "$sort"]
+ [:ref ::sort-spec]])
+
+(mr/def ::$match-stage
+ [:map-of
+ [:= "$match"]
+ [:map-of
+ [:and
+ [:or ::lib.schema.common/non-blank-string :keyword]
+ [:fn
+ {:error/message "not a $not condition"}
+ (complement #{:$not "$not"})]]
+ :any]])
+
+(mr/def ::$group-stage
+ [:map-of
+ [:= "$group"]
+ [:map-of ::lib.schema.common/non-blank-string :any]])
+
+(mr/def ::$add-fields-stage
+ [:map-of
+ [:= "$addFields"]
+ [:map-of ::lib.schema.common/non-blank-string :any]])
+
+(mr/def ::$lookup-stage
+ [:map-of
+ [:= "$lookup"]
+ [:map
+ {:closed true} ; add more stuff as needed
+ [:from :string]
+ [:let [:map
+ [:vars {:optional true} :any]
+ [:in {:optional true} :any]]]
+ [:pipeline [:ref ::pipeline]]
+ [:as :string]]])
+
+(mr/def ::$unwind-stage
+ [:map-of
+ [:= "$unwind"]
+ [:map-of [:or :keyword :string] :any]])
+
+(mr/def ::$limit-stage
+ [:map-of
+ [:= "$limit"]
+ pos-int?])
+
+(mr/def ::$skip-stage
+ [:map-of
+ [:= "$skip"]
+ pos-int?])
+
+(mr/def ::$set-window-fields-stage
+ [:map-of
+ [:= "$setWindowFields"]
+ [:map
+ {:closed true} ; add more stuff as needed
+ ["sortBy" [:ref ::sort-spec]]
+ ["output" [:map-of
+ ::lib.schema.common/non-blank-string
+ [:map-of ::lib.schema.common/non-blank-string :any]]]
+ ["partitionBy" {:optional true} [:map-of
+ ::lib.schema.common/non-blank-string
+ ::lib.schema.common/non-blank-string]]]])
+
+(defn- contains-uncompiled-mbql-clause? [x]
+ (cond
+ (map? x)
+ (or (contains? x :lib/uuid)
+ (some contains-uncompiled-mbql-clause? (vals x)))
+
+ (sequential? x)
+ (some contains-uncompiled-mbql-clause? x)
+
+ :else
+ false))
+
+(mr/def ::no-uncompiled-mbql
+ [:fn
+ {:error/message "Should not contain uncompiled MBQL clauses"}
+ (complement contains-uncompiled-mbql-clause?)])
+
+(mr/def ::stage
+ [:multi
+ {:dispatch #(boolean (instance? org.bson.Document %))}
+ [true
+ ;; TODO (Cam 2026-07-17) consider whether we should parsed the BSON documents out into Ordered Maps so we can
+ ;; properly validate them... even tho they implement `java.util.Map` Malli won't validate them as `:map` because
+ ;; they're not `map?`
+ (lib.schema.common/instance-of-class org.bson.Document)]
+ [false
+ [:and
+ :map
+ [:fn
+ {:error/message "map with a single key"}
+ #(= (count %) 1)]
+ [:multi
+ {:dispatch (fn [m]
+ (when (map? m)
+ (first (keys m))))}
+ ["$project" ::$project-stage]
+ ["$sort" ::$sort-stage]
+ ["$group" ::$group-stage]
+ ["$addFields" ::$add-fields-stage]
+ ["$lookup" ::$lookup-stage]
+ ["$unwind" ::$unwind-stage]
+ ["$match" ::$match-stage]
+ ["$limit" ::$limit-stage]
+ ["$skip" ::$skip-stage]
+ ["$setWindowFields" ::$set-window-fields-stage]]
+ [:ref ::no-uncompiled-mbql]]]])
+
+(mr/def ::pipeline
+ [:sequential ::stage])
+
+(mr/def ::projection
+ :string)
+
+(mr/def ::projections
"Schema for the `:projections` generated by the functions in this namespace. It is a sequence of the column names
returned in an MBQL query. e.g.
[\"_id\" \"date\" \"user_id\" \"venue_id\"]"
- [:sequential :string])
+ [:and
+ [:sequential ::projection]
+ [:fn
+ {:error/message "projected column names should be distinct"}
+ (fn [coll]
+ (if (seq coll)
+ (apply distinct? coll)
+ true))]])
+
+(mr/def ::compiled-pipeline
+ "Compiled pipeline query. Note that this is actually a subset of `:metabase.query-processor.compile/compiled`.
+
+ This is also the schema for the value of `:native` in a MBQL stage for MongoDB."
+ [:merge
+ ;; `:query` (native) and `:params` (like JDBC params; empty for MongoDB)
+ [:ref :metabase.query-processor.compile/compiled]
+ [:map
+ {:closed true} ; we should document anything else we add here.
+ [:projections {:optional true} [:maybe ::projections]]
+ [:query [:ref ::pipeline]]
+ ;; TODO (Cam 2026-07-17) it's not really clear if `:collection` is supposed to be in the top-level of the stage e.g.
+ ;;
+ ;; {:lib/type :mbql.stage/native, :collection "X", :native {...}}
+ ;;
+ ;; or within `:native` e.g.
+ ;;
+ ;; {:lib/type :mbql.stage/native, :native {:collection "X", ...}}
+ ;;
+ ;; [[metabase.query-processor.middleware.fetch-source-query/fix-mongodb-first-stage]] seems to put it in `:native`
+ ;; itself but the [[metabase.driver/execute-reducible-query]] implementation for
+ ;; MongoDB (in [[metabase.driver.mongo]]) assumes it's in the top level.
+ ;;
+ ;; I'm not clear where the FE sets it either.
+ ;;
+ ;; Let's standardize on one or the other and update the Lib schema to enforce the key being in the right
+ ;; place (normalizing if needed).
+ [:collection {:optional true} :string]
+ ;; whether this was compiled from MBQL or not (`false` means the query was native and never compiled from MBQL)
+ [:mbql? {:optional true} [:maybe :boolean]]]])
;;; +----------------------------------------------------------------------------------------------------------------+
;;; | QP Impl |
;;; +----------------------------------------------------------------------------------------------------------------+
-;; TODO - We already have a *query* dynamic var in metabase.query-processor.interface. Do we need this one too?
-(def ^:dynamic ^:private *query* nil)
-
-(def ^:dynamic ^:private *nesting-level*
- "Used for tracking depth of nesting on which [[mbql->native-rec]] operates.
- That is required eg. in `->lvalue :aggregation` call."
- 0)
-
(def ^:dynamic ^:private *next-alias-index*
"Tracks index of next alias for join compilation. It is bound in [[mbql->native]] to `volatile!` valued 0. Hence
every compilation starts with a fresh 0. Indices are used in [[handle-join]] to make aliases unique. Index values
@@ -114,26 +244,39 @@
[]
(vswap! *next-alias-index* inc))
+;; TODO (Cam 2026-07-24) get rid of this dynamic var and attach the mappings directly to the `query` itself
(def ^:dynamic ^:private *field-mappings*
"The mapping from the fields to the projected names created
by the nested query."
{})
-(defn- find-mapped-field-name
+(mu/defn- find-mapped-field-name
"Finds the name of a mapped field, if any.
First it does a quick exact match and if the field is not found, it searches for a field with the same ID/name and
the same join alias.
Note that during the compilation of joins, the field :join-alias is renamed to ::join-local to prevent prefixing the
fields of the current join to be prefixed with the join alias."
- [[_ field-id params :as field]]
- (or (get *field-mappings* field)
- (some (fn [[e n]]
- (when (and (vector? e)
- (= (subvec e 0 2) [:field field-id])
- (= (:join-alias (e 2)) (:join-alias params))
- (= (::join-local (e 2)) (::join-local params)))
- n))
- *field-mappings*)))
+ [[_ opts field-id :as field-ref] :- :mbql.clause/field]
+ (letfn [(lib-match? [a-ref]
+ (and (lib.equality/= a-ref field-ref)
+ (= (::join-local (lib/options a-ref)) (::join-local opts))))
+ ;; TODO (Cam 2026-08-11) Riley pointed out that this check is probably wrong if we have two breakouts on the
+ ;; same field (e.g. with different temporal units), altho this is the old behavior from the pre-MBQL 5
+ ;; version of the MongoDB driver and somehow things still worked... I'm leaving it in place for now because
+ ;; I don't fully understand what's going on here, maybe we're mapping multiple breakouts on the same field
+ ;; to the same thing anyway
+ (fallback-match? [a-ref]
+ (and (vector? a-ref)
+ (= (first a-ref) :field)
+ (= (last a-ref) field-id)
+ (= (:join-alias (lib/options a-ref)) (:join-alias opts))
+ (= (::join-local (lib/options a-ref)) (::join-local opts))))
+ (matching-name [match?]
+ (some (fn [[a-ref mapped-name]]
+ (when (match? a-ref)
+ mapped-name))
+ *field-mappings*))]
+ (some matching-name [lib-match? fallback-match?])))
(defn- get-join-alias
"Calculates the name of the join field used for `join-alias`, if any.
@@ -143,22 +286,31 @@
[join-alias]
(some->> join-alias (str "join_alias_")))
-(defn- get-mongo-version []
+(mu/defn- get-mongo-version
+ [metadata-providerable :- ::lib.schema.metadata/metadata-providerable]
(driver-api/cached ::version
- (driver/dbms-version :mongo (driver-api/database (driver-api/metadata-provider)))))
+ (driver/dbms-version :mongo (driver-api/database metadata-providerable))))
(defmulti ^:private ->rvalue
- "Format this `Field` or value for use as the right hand value of an expression, e.g. by adding `$` to a `Field`'s
+ "Format this MBQL clause or value for use as the right hand value of an expression, e.g. by adding `$` to a `Field`'s
name"
- {:arglists '([x])}
- driver-api/dispatch-by-clause-name-or-class)
+ {:arglists '([query stage-number x])}
+ (fn [_query _stage-number x]
+ (when (vector? x)
+ (when-not (lib/clause? x)
+ (throw (ex-info "Attempted to call ->rvalue on a vector that is not an MBQL clause!"
+ {:x x}))))
+ (driver-api/dispatch-by-clause-name-or-class x)))
(defmulti ^:private ->lvalue
"Return an escaped name that can be used as the name of a given Field."
- {:arglists '([field])}
- driver-api/dispatch-by-clause-name-or-class)
+ {:arglists '([query stage-number field])}
+ (fn [_query _stage-number field]
+ (driver-api/dispatch-by-clause-name-or-class field)))
-(defn- col->name-components [{:keys [parent-id nfc-path], field-name :name, :as _col}]
+(mu/defn- col->name-components :- [:sequential :string]
+ [metadata-providerable :- ::lib.schema.metadata/metadata-providerable
+ {:keys [parent-id nfc-path], field-name :name, :as _col} :- ::lib.schema.metadata/column]
(cond
;; mongo sync stores `:nfc-path` with the field's own name as the last element, matching sql-jdbc nested json
(seq nfc-path)
@@ -166,7 +318,7 @@
;; fall back to walking `:parent-id` for fields synced before `:nfc-path` was populated
parent-id
- (concat (col->name-components (driver-api/field (driver-api/metadata-provider) parent-id))
+ (concat (col->name-components metadata-providerable (driver-api/field metadata-providerable parent-id))
[field-name])
:else
@@ -181,12 +333,13 @@
(mu/defn field->name
"Return a single string name for column metadata `col` For nested fields, this creates a combined qualified name."
- ([col]
- (field->name col \.))
+ ([metadata-providerable col]
+ (field->name metadata-providerable col \.))
- ([col :- driver-api/schema.metadata.column
- separator :- [:or :string char?]]
- (str/join separator (col->name-components col))))
+ ([metadata-providerable :- ::lib.schema.metadata/metadata-providerable
+ col :- ::lib.schema.metadata/column
+ separator :- [:or :string char?]]
+ (str/join separator (col->name-components metadata-providerable col))))
(defmacro ^:private mongo-let
{:style/indent 1}
@@ -197,31 +350,36 @@
(declare with-rvalue-temporal-bucketing)
-(defn- scope-with-join-field
+(mu/defn- scope-with-join-field :- :string
"Adjust `field-name` for fields coming from joins. For use in `->[lr]value` for `:field` and `:metadata/column`."
- [field-name join-field source-alias]
+ [field-name :- [:maybe :string]
+ join-field :- [:maybe :string]
+ source-alias :- [:maybe :string]]
(cond->> (or source-alias field-name)
join-field (str join-field \.)))
(defmethod ->lvalue :metadata/column
- [{::keys [join-field source-alias] :as field}]
- (scope-with-join-field (field->name field) join-field source-alias))
+ [query _stage-number {::keys [join-field source-alias] :as field}]
+ (scope-with-join-field (field->name query field) join-field source-alias))
-(defmethod ->lvalue :expression
- [[_ expression-name opts]]
+(mu/defmethod ->lvalue :expression
+ [_query _stage-number [_ opts expression-name] :- :mbql.clause/expression]
(or (get opts driver-api/qp.add.desired-alias) expression-name))
-(defmethod ->rvalue :default
- [x]
+(mu/defmethod ->rvalue :default
+ [_query _stage-number x]
x)
-(defmethod ->rvalue :expression
- [[_ expression-name {:keys [temporal-unit]}]]
- (let [expression-value (driver-api/expression-with-name (:query *query*) expression-name)
- rvalue (cond->> (->rvalue expression-value)
+(mu/defmethod ->rvalue :expression
+ [query :- ::lib.schema/query
+ stage-number :- :int
+ [_ {:keys [temporal-unit]} expression-name] :- :mbql.clause/expression]
+ (let [expression-value (lib/resolve-expression query stage-number expression-name)
+ rvalue (cond->> (->rvalue query stage-number expression-value)
(driver-api/is-clause? :value expression-value) (array-map $literal))]
- (cond-> rvalue
- temporal-unit (with-rvalue-temporal-bucketing temporal-unit))))
+ (if temporal-unit
+ (with-rvalue-temporal-bucketing query rvalue temporal-unit)
+ rvalue)))
(def ^:private base64-decoder "
function(bin) {
@@ -258,9 +416,11 @@ function(bin) {
}
")
-(defmethod ->rvalue :metadata/column
- [{coercion :coercion-strategy, ::keys [source-alias join-field inherited?] :as field}]
- (let [field-name (str \$ (scope-with-join-field (field->name field) join-field source-alias))
+(mu/defmethod ->rvalue :metadata/column
+ [query
+ _stage-number
+ {coercion :coercion-strategy, ::keys [source-alias join-field inherited?] :as field} :- ::lib.schema.metadata/column]
+ (let [field-name (str \$ (scope-with-join-field (field->name query field) join-field source-alias))
coercion (when-not inherited?
coercion)]
(cond
@@ -312,7 +472,7 @@ function(bin) {
:coercion-strategy coercion}))
(isa? coercion :Coercion/DateTime->Date)
- (with-rvalue-temporal-bucketing field-name :day)
+ (with-rvalue-temporal-bucketing query field-name :day)
(isa? coercion :Coercion/String->Float)
{"$toDouble" field-name}
@@ -329,17 +489,17 @@ function(bin) {
;;
;; aggregations[0] = 20
;;
-(defmethod ->lvalue :aggregation
- [[_ index]]
- (driver-api/aggregation-name (:query *query*) (driver-api/aggregation-at-index *query* index *nesting-level*)))
+(mu/defmethod ->lvalue :aggregation
+ [query stage-number ag-ref :- :mbql.clause/aggregation]
+ (driver-api/mbql-5-aggregation-name query stage-number ag-ref))
-(defmethod ->lvalue :field
- [[_ id-or-name {:keys [join-alias] :as opts} :as field]]
- (if (integer? id-or-name)
+(mu/defmethod ->lvalue :field
+ [query stage-number [_ {:keys [join-alias] :as opts} id-or-name :as field] :- :mbql.clause/field]
+ (if (pos-int? id-or-name)
(or (find-mapped-field-name field)
- (->lvalue (assoc (driver-api/field (driver-api/metadata-provider) id-or-name)
- ::source-alias (driver-api/qp.add.source-alias opts)
- ::join-field (get-join-alias join-alias))))
+ (->lvalue query stage-number (assoc (driver-api/field query id-or-name)
+ ::source-alias (driver-api/qp.add.source-alias opts)
+ ::join-field (get-join-alias join-alias))))
(scope-with-join-field (name id-or-name) (get-join-alias join-alias) (driver-api/qp.add.source-alias opts))))
(defn- add-start-of-week-offset [expr offset]
@@ -373,34 +533,43 @@ function(bin) {
[resolution])]
[part (str (name parts) \. (name part))]))}))
-(defn- days-till-start-of-first-full-week
- [column]
- (let [start-of-year (with-rvalue-temporal-bucketing column :year)
- day-of-week-of-start-of-year (with-rvalue-temporal-bucketing start-of-year :day-of-week)]
+(mu/defn- days-till-start-of-first-full-week
+ [metadata-providerable :- ::lib.schema.metadata/metadata-providerable
+ column]
+ (let [start-of-year (with-rvalue-temporal-bucketing metadata-providerable column :year)
+ day-of-week-of-start-of-year (with-rvalue-temporal-bucketing metadata-providerable start-of-year :day-of-week)]
{:$subtract [8 day-of-week-of-start-of-year]}))
-(defn- week-of-year
+(mu/defn- week-of-year
"Full explanation of this magic is in [[metabase.driver.sql.query-processor/week-of-year]]."
- [column mode]
- (let [doy (with-rvalue-temporal-bucketing column :day-of-year)
+ [metadata-providerable :- ::lib.schema.metadata/metadata-providerable
+ column
+ mode]
+ (let [doy (with-rvalue-temporal-bucketing metadata-providerable column :day-of-year)
dtsofw (binding [driver.common/*start-of-week* (case mode
:us :sunday
:instance nil)]
- (days-till-start-of-first-full-week column))]
+ (days-till-start-of-first-full-week metadata-providerable column))]
{:$toInt {:$add [1 {:$ceil {:$divide [{:$subtract [doy dtsofw]} 7]}}]}}))
(defn- extract
[op column]
{op {:date column :timezone (driver-api/results-timezone-id)}})
-(defn- with-rvalue-temporal-bucketing
- [field unit]
+(mu/defn- with-rvalue-temporal-bucketing
+ [metadata-providerable :- ::lib.schema.metadata/metadata-providerable
+ column
+ ;; TODO (Cam 2026-07-24) apparently there's no complete schema with all the valid bucketing and truncation
+ ;; units (!) fix this
+ unit :- [:or
+ ::lib.schema.temporal-bucketing/unit
+ ::lib.schema.expression.temporal/temporal-extract.unit]]
+ {:style/indent [:form]}
(if (= unit :default)
- field
- (let [supports-dateTrunc? (-> (get-mongo-version)
+ column
+ (let [supports-dateTrunc? (-> (get-mongo-version metadata-providerable)
:semantic-version
- (driver.u/semantic-version-gte [5]))
- column field]
+ (driver.u/semantic-version-gte [5]))]
(letfn [(truncate [unit]
(if supports-dateTrunc?
{:$dateTrunc {:date column
@@ -430,8 +599,8 @@ function(bin) {
{:$ceil {$divide [{$dayOfYear week-start}
7.0]}})
:week-of-year-iso (extract :$isoWeek column)
- :week-of-year-us (week-of-year column :us)
- :week-of-year-instance (week-of-year column :instance)
+ :week-of-year-us (week-of-year metadata-providerable column :us)
+ :week-of-year-instance (week-of-year metadata-providerable column :instance)
:month (truncate :month)
:month-of-year (extract $month column)
;; For quarter we'll just subtract enough days from the current date to put it in the correct month and
@@ -456,26 +625,32 @@ function(bin) {
:year-of-era
(extract $year column))))))
-(defmethod ->rvalue :field
- [[_ id-or-name {:keys [temporal-unit join-alias] :as opts} :as field]]
+(mu/defmethod ->rvalue :field
+ [query :- ::lib.schema/query
+ stage-number :- :int
+ [_ {:keys [temporal-unit join-alias] :as opts} id-or-name :as field] :- :mbql.clause/field]
(let [join-field (get-join-alias join-alias)
- source-alias (driver-api/qp.add.source-alias opts)]
- (cond-> (if (integer? id-or-name)
- (if-let [mapped (find-mapped-field-name field)]
- (str \$ mapped)
- (->rvalue (assoc (driver-api/field (driver-api/metadata-provider) id-or-name)
- ::source-alias source-alias
- ::join-field join-field
- ::inherited? (not (or (pos-int? (driver-api/qp.add.source-table opts))
- (:qp/allow-coercion-for-columns-without-integer-qp.add.source-table opts))))))
- (if-let [mapped (find-mapped-field-name field)]
- (str \$ mapped)
- (str \$ (scope-with-join-field (name id-or-name) join-field source-alias))))
- temporal-unit (with-rvalue-temporal-bucketing temporal-unit))))
+ source-alias (driver-api/qp.add.source-alias opts)
+ rvalue (if (pos-int? id-or-name)
+ (if-let [mapped (find-mapped-field-name field)]
+ (str \$ mapped)
+ (->rvalue query stage-number (assoc (driver-api/field query id-or-name)
+ ::source-alias source-alias
+ ::join-field join-field
+ ::inherited? (not (or (pos-int? (driver-api/qp.add.source-table opts))
+ (:qp/allow-coercion-for-columns-without-integer-qp.add.source-table opts))))))
+ (if-let [mapped (find-mapped-field-name field)]
+ (str \$ mapped)
+ (str \$ (scope-with-join-field (name id-or-name) join-field source-alias))))]
+ (if temporal-unit
+ (with-rvalue-temporal-bucketing query rvalue temporal-unit)
+ rvalue)))
;; Values clauses below; they only need to implement `->rvalue`
-(defmethod ->rvalue nil [_] nil)
+(mu/defmethod ->rvalue nil
+ [_query _stage-number _nil]
+ nil)
(defn- uuid->bsonbinary
[u]
@@ -487,8 +662,8 @@ function(bin) {
(.array))]
(Binary. BsonBinarySubType/UUID_STANDARD ba)))
-(defmethod ->rvalue :value
- [[_ value {base-type :base_type}]]
+(mu/defmethod ->rvalue :value
+ [_query _stage-number [_ {base-type :base-type} value] :- :mbql.clause/value]
(cond
;; Passing nil or "" to the ObjectId or Binary constructor throws an exception
(or (nil? value) (= value ""))
@@ -516,17 +691,21 @@ function(bin) {
(defn- $date-from-string [s]
{:$dateFromString {:dateString (str s)}})
-(defmethod ->rvalue :absolute-datetime
- [[_ t unit]]
- (let [report-zone (t/zone-id (or (driver-api/report-timezone-id-if-supported :mongo (driver-api/database (driver-api/metadata-provider)))
+(mu/defn- absolute-datetime-or-time->rvalue
+ [metadata-providerable
+ [_ _opts t unit] :- [:or :mbql.clause/absolute-datetime :mbql.clause/time]]
+ (let [report-zone (t/zone-id (or (driver-api/report-timezone-id-if-supported :mongo (driver-api/database metadata-providerable))
"UTC"))
- t (condp = (class t)
- java.time.LocalDate t
- java.time.LocalTime t
- java.time.LocalDateTime t
- java.time.OffsetTime (t/offset-time t report-zone)
- java.time.OffsetDateTime (t/offset-date-time t report-zone)
- java.time.ZonedDateTime (t/offset-date-time t report-zone))]
+ normalize-t (fn [t]
+ (condp = (class t)
+ String (recur (u.date/parse t))
+ java.time.LocalDate t
+ java.time.LocalTime t
+ java.time.LocalDateTime t
+ java.time.OffsetTime (t/offset-time t report-zone)
+ java.time.OffsetDateTime (t/offset-date-time t report-zone)
+ java.time.ZonedDateTime (t/offset-date-time t report-zone)))
+ t (normalize-t t)]
(letfn [(extract [unit]
(u.date/extract t unit))
(bucket [unit]
@@ -549,10 +728,18 @@ function(bin) {
:quarter-of-year (extract :quarter-of-year)
:year (bucket :year)))))
-(defmethod ->rvalue :relative-datetime
- [[_ amount unit]]
+(mu/defmethod ->rvalue :absolute-datetime
+ [query _stage-number clause :- :mbql.clause/absolute-datetime]
+ (absolute-datetime-or-time->rvalue query clause))
+
+(mu/defmethod ->rvalue :time
+ [query _stage-number clause :- :mbql.clause/time]
+ (absolute-datetime-or-time->rvalue query clause))
+
+(mu/defmethod ->rvalue :relative-datetime
+ [query _stage-number [_ _opts amount unit] :- :mbql.clause/relative-datetime]
(let [t (-> (t/zoned-date-time)
- (t/with-zone-same-instant (t/zone-id (or (driver-api/report-timezone-id-if-supported :mongo (driver-api/database (driver-api/metadata-provider)))
+ (t/with-zone-same-instant (t/zone-id (or (driver-api/report-timezone-id-if-supported :mongo (driver-api/database query))
"UTC"))))]
($date-from-string
(t/offset-date-time
@@ -566,90 +753,147 @@ function(bin) {
;; It doesn't make 100% sense to have lvalues for all these but it's a formal requirement
-(defmethod ->lvalue :avg [[_ inp]] (->lvalue inp))
-(defmethod ->lvalue :stddev [[_ inp]] (->lvalue inp))
-(defmethod ->lvalue :var [[_ inp]] (->lvalue inp))
-(defmethod ->lvalue :sum [[_ inp]] (->lvalue inp))
-(defmethod ->lvalue :min [[_ inp]] (->lvalue inp))
-(defmethod ->lvalue :max [[_ inp]] (->lvalue inp))
-
-(defmethod ->lvalue :floor [[_ inp]] (->lvalue inp))
-(defmethod ->lvalue :ceil [[_ inp]] (->lvalue inp))
-(defmethod ->lvalue :round [[_ inp]] (->lvalue inp))
-(defmethod ->lvalue :abs [[_ inp]] (->lvalue inp))
-
-(defmethod ->lvalue :log [[_ inp]] (->lvalue inp))
-(defmethod ->lvalue :exp [[_ inp]] (->lvalue inp))
-(defmethod ->lvalue :sqrt [[_ inp]] (->lvalue inp))
-
-(defmethod ->lvalue :trim [[_ inp]] (->lvalue inp))
-(defmethod ->lvalue :ltrim [[_ inp]] (->lvalue inp))
-(defmethod ->lvalue :rtrim [[_ inp]] (->lvalue inp))
-(defmethod ->lvalue :upper [[_ inp]] (->lvalue inp))
-(defmethod ->lvalue :lower [[_ inp]] (->lvalue inp))
-(defmethod ->lvalue :length [[_ inp]] (->lvalue inp))
-
-(defmethod ->lvalue :power [[_ & args]] (->lvalue (first args)))
-(defmethod ->lvalue :replace [[_ & args]] (->lvalue (first args)))
-(defmethod ->lvalue :concat [[_ & args]] (->lvalue (first args)))
-(defmethod ->lvalue :substring [[_ & args]] (->lvalue (first args)))
-
-(defmethod ->lvalue :+ [[_ & args]] (->lvalue (first args)))
-(defmethod ->lvalue :- [[_ & args]] (->lvalue (first args)))
-(defmethod ->lvalue :* [[_ & args]] (->lvalue (first args)))
-(defmethod ->lvalue :/ [[_ & args]] (->lvalue (first args)))
-
-(defmethod ->lvalue :coalesce [[_ & args]] (->lvalue (first args)))
-
-(defmethod ->rvalue :avg [[_ inp]] {$avg (->rvalue inp)})
-(defmethod ->rvalue :stddev [[_ inp]] {"$stdDevSamp" (->rvalue inp)})
-(defmethod ->rvalue :sum [[_ inp]] {"$sum" (->rvalue inp)})
-(defmethod ->rvalue :min [[_ inp]] {$min (->rvalue inp)})
-(defmethod ->rvalue :max [[_ inp]] {$max (->rvalue inp)})
-
-(defmethod ->rvalue :floor [[_ inp]] {"$floor" (->rvalue inp)})
-(defmethod ->rvalue :ceil [[_ inp]] {"$ceil" (->rvalue inp)})
-(defmethod ->rvalue :round [[_ inp]] {"$round" (->rvalue inp)})
-(defmethod ->rvalue :abs [[_ inp]] {"$abs" (->rvalue inp)})
-
-(defmethod ->rvalue :log [[_ inp]] {"$log10" (->rvalue inp)})
-(defmethod ->rvalue :exp [[_ inp]] {"$exp" (->rvalue inp)})
-(defmethod ->rvalue :sqrt [[_ inp]] {"$sqrt" (->rvalue inp)})
-
-(defmethod ->rvalue :trim [[_ inp]] {"$trim" {"input" (->rvalue inp)}})
-(defmethod ->rvalue :ltrim [[_ inp]] {"$ltrim" {"input" (->rvalue inp)}})
-(defmethod ->rvalue :rtrim [[_ inp]] {"$rtrim" {"input" (->rvalue inp)}})
-(defmethod ->rvalue :upper [[_ inp]] {"$toUpper" (->rvalue inp)})
-(defmethod ->rvalue :lower [[_ inp]] {"$toLower" (->rvalue inp)})
-(defmethod ->rvalue :length [[_ inp]] {"$strLenCP" (->rvalue inp)})
-
-(defmethod ->rvalue :power [[_ & args]] {"$pow" (mapv ->rvalue args)})
-(defmethod ->rvalue :concat [[_ & args]] {"$concat" (mapv ->rvalue args)})
-(defmethod ->rvalue :temporal-extract [[_ inp unit]]
- (with-rvalue-temporal-bucketing (->rvalue inp) unit))
-
-(defmethod ->rvalue :replace
- [[_ & args]]
- (let [version (get-mongo-version)]
+(defmethod ->lvalue :avg [query stage-number [_ _opts inp]] (->lvalue query stage-number inp))
+(defmethod ->lvalue :stddev [query stage-number [_ _opts inp]] (->lvalue query stage-number inp))
+(defmethod ->lvalue :var [query stage-number [_ _opts inp]] (->lvalue query stage-number inp))
+(defmethod ->lvalue :sum [query stage-number [_ _opts inp]] (->lvalue query stage-number inp))
+(defmethod ->lvalue :min [query stage-number [_ _opts inp]] (->lvalue query stage-number inp))
+(defmethod ->lvalue :max [query stage-number [_ _opts inp]] (->lvalue query stage-number inp))
+
+(defmethod ->lvalue :floor [query stage-number [_ _opts inp]] (->lvalue query stage-number inp))
+(defmethod ->lvalue :ceil [query stage-number [_ _opts inp]] (->lvalue query stage-number inp))
+(defmethod ->lvalue :round [query stage-number [_ _opts inp]] (->lvalue query stage-number inp))
+(defmethod ->lvalue :abs [query stage-number [_ _opts inp]] (->lvalue query stage-number inp))
+
+(defmethod ->lvalue :log [query stage-number [_ _opts inp]] (->lvalue query stage-number inp))
+(defmethod ->lvalue :exp [query stage-number [_ _opts inp]] (->lvalue query stage-number inp))
+(defmethod ->lvalue :sqrt [query stage-number [_ _opts inp]] (->lvalue query stage-number inp))
+
+(defmethod ->lvalue :trim [query stage-number [_ _opts inp]] (->lvalue query stage-number inp))
+(defmethod ->lvalue :ltrim [query stage-number [_ _opts inp]] (->lvalue query stage-number inp))
+(defmethod ->lvalue :rtrim [query stage-number [_ _opts inp]] (->lvalue query stage-number inp))
+(defmethod ->lvalue :upper [query stage-number [_ _opts inp]] (->lvalue query stage-number inp))
+(defmethod ->lvalue :lower [query stage-number [_ _opts inp]] (->lvalue query stage-number inp))
+(defmethod ->lvalue :length [query stage-number [_ _opts inp]] (->lvalue query stage-number inp))
+
+(defmethod ->lvalue :power [query stage-number [_ _opts & args]] (->lvalue query stage-number (first args)))
+(defmethod ->lvalue :replace [query stage-number [_ _opts & args]] (->lvalue query stage-number (first args)))
+(defmethod ->lvalue :concat [query stage-number [_ _opts & args]] (->lvalue query stage-number (first args)))
+(defmethod ->lvalue :substring [query stage-number [_ _opts & args]] (->lvalue query stage-number (first args)))
+
+(defmethod ->lvalue :+ [query stage-number [_ _opts & args]] (->lvalue query stage-number (first args)))
+(defmethod ->lvalue :- [query stage-number [_ _opts & args]] (->lvalue query stage-number (first args)))
+(defmethod ->lvalue :* [query stage-number [_ _opts & args]] (->lvalue query stage-number (first args)))
+(defmethod ->lvalue :/ [query stage-number [_ _opts & args]] (->lvalue query stage-number (first args)))
+
+(defmethod ->lvalue :coalesce [query stage-number [_ _opts & args]] (->lvalue query stage-number (first args)))
+
+(mu/defmethod ->rvalue :avg
+ [query stage-number [_ _opts inp] :- :mbql.clause/avg]
+ {$avg (->rvalue query stage-number inp)})
+
+(mu/defmethod ->rvalue :stddev
+ [query stage-number [_ _opts expr] :- :mbql.clause/stddev]
+ {"$stdDevSamp" (->rvalue query stage-number expr)})
+
+(mu/defmethod ->rvalue :sum
+ [query stage-number [_ _opts inp] :- :mbql.clause/sum]
+ {"$sum" (->rvalue query stage-number inp)})
+
+(mu/defmethod ->rvalue :min
+ [query stage-number [_ _opts inp] :- :mbql.clause/min]
+ {$min (->rvalue query stage-number inp)})
+
+(mu/defmethod ->rvalue :max
+ [query stage-number [_ _opts inp] :- :mbql.clause/max]
+ {$max (->rvalue query stage-number inp)})
+
+(mu/defmethod ->rvalue :floor
+ [query stage-number [_ _opts inp] :- :mbql.clause/floor]
+ {"$floor" (->rvalue query stage-number inp)})
+
+(mu/defmethod ->rvalue :ceil
+ [query stage-number [_ _opts inp] :- :mbql.clause/ceil]
+ {"$ceil" (->rvalue query stage-number inp)})
+
+(mu/defmethod ->rvalue :round
+ [query stage-number [_ _opts inp] :- :mbql.clause/round]
+ {"$round" (->rvalue query stage-number inp)})
+
+(mu/defmethod ->rvalue :abs
+ [query stage-number [_ _opts inp] :- :mbql.clause/abs]
+ {"$abs" (->rvalue query stage-number inp)})
+
+(mu/defmethod ->rvalue :log
+ [query stage-number [_ _opts inp] :- :mbql.clause/log]
+ {"$log10" (->rvalue query stage-number inp)})
+
+(mu/defmethod ->rvalue :exp
+ [query stage-number [_ _opts inp] :- :mbql.clause/exp]
+ {"$exp" (->rvalue query stage-number inp)})
+
+(mu/defmethod ->rvalue :sqrt
+ [query stage-number [_ _opts inp] :- :mbql.clause/sqrt]
+ {"$sqrt" (->rvalue query stage-number inp)})
+
+(mu/defmethod ->rvalue :trim
+ [query stage-number [_ _opts inp] :- :mbql.clause/trim]
+ {"$trim" {"input" (->rvalue query stage-number inp)}})
+
+(mu/defmethod ->rvalue :ltrim
+ [query stage-number [_ _opts inp] :- :mbql.clause/ltrim]
+ {"$ltrim" {"input" (->rvalue query stage-number inp)}})
+
+(mu/defmethod ->rvalue :rtrim
+ [query stage-number [_ _opts inp] :- :mbql.clause/rtrim]
+ {"$rtrim" {"input" (->rvalue query stage-number inp)}})
+
+(mu/defmethod ->rvalue :upper
+ [query stage-number [_ _opts inp] :- :mbql.clause/upper]
+ {"$toUpper" (->rvalue query stage-number inp)})
+
+(mu/defmethod ->rvalue :lower
+ [query stage-number [_ _opts inp] :- :mbql.clause/lower]
+ {"$toLower" (->rvalue query stage-number inp)})
+
+(mu/defmethod ->rvalue :length
+ [query stage-number [_ _opts inp] :- :mbql.clause/length]
+ {"$strLenCP" (->rvalue query stage-number inp)})
+
+(mu/defmethod ->rvalue :power
+ [query stage-number [_ _opts & args] :- :mbql.clause/power]
+ {"$pow" (mapv (partial ->rvalue query stage-number) args)})
+
+(mu/defmethod ->rvalue :concat
+ [query stage-number [_ _opts & args] :- :mbql.clause/concat]
+ {"$concat" (mapv (partial ->rvalue query stage-number) args)})
+
+(mu/defmethod ->rvalue :temporal-extract
+ [query stage-number [_ _opts inp unit] :- :mbql.clause/temporal-extract]
+ (with-rvalue-temporal-bucketing query (->rvalue query stage-number inp) unit))
+
+(mu/defmethod ->rvalue :replace
+ [query stage-number [_ _opts & args] :- :mbql.clause/replace]
+ (let [version (get-mongo-version query)]
(if (driver.u/semantic-version-gte (:semantic-version version) [4 4])
- (let [[expr fnd replacement] (mapv ->rvalue args)]
+ (let [[expr fnd replacement] (mapv (partial ->rvalue query stage-number) args)]
{"$replaceAll" {"input" expr "find" fnd "replacement" replacement}})
(throw (ex-info "Replace requires MongoDB 4.4 or above"
{:database-version version})))))
-(defmethod ->rvalue :substring
- [[_ & [expr idx cnt]]]
- (let [expr-val (->rvalue expr)
- idx-val {"$subtract" [(->rvalue idx) 1]}]
+(mu/defmethod ->rvalue :substring
+ [query stage-number [_ _opts string-expr start length] :- :mbql.clause/substring]
+ (let [expr-val (->rvalue query stage-number string-expr)
+ idx-val {"$subtract" [(->rvalue query stage-number start) 1]}]
{"$substrCP" [expr-val
idx-val
;; The last argument is not optional in mongo
- (if (some? cnt)
- (->rvalue cnt)
+ (if (some? length)
+ (->rvalue query stage-number length)
{"$subtract" [{"$strLenCP" expr-val} idx-val]})]}))
-(defmethod ->rvalue :/
- [[_ & [_ & divisors :as args]]]
+(mu/defmethod ->rvalue :/
+ [query stage-number [_ _opts & [_ & divisors :as args]] :- :mbql.clause//]
;; division works outside in (/ 1 2 3) => (/ (/ 1 2) 3)
(let [division (reduce
(fn [accum head]
@@ -657,9 +901,9 @@ function(bin) {
{"$divide" [accum head]}
head))
nil
- (map ->rvalue args))
+ (map (partial ->rvalue query stage-number) args))
literal-zero? (some #(and (number? %) (zero? %)) divisors)
- non-literal-nil-checks (mapv (fn [divisor] {"$eq" [(->rvalue divisor) 0]}) (remove number? divisors))]
+ non-literal-nil-checks (mapv (fn [divisor] {"$eq" [(->rvalue query stage-number divisor) 0]}) (remove number? divisors))]
(cond
literal-zero?
nil
@@ -683,8 +927,8 @@ function(bin) {
;;; Because of this, whenever we translate date arithmetic with intervals, we check the major
;;; version of the database and throw a nice exception if it's less than 5.
-(defn- check-date-operations-supported []
- (let [{mongo-version :version, [major-version] :semantic-version} (get-mongo-version)]
+(defn- check-date-operations-supported [metadata-providerable]
+ (let [{mongo-version :version, [major-version] :semantic-version} (get-mongo-version metadata-providerable)]
(when (and major-version (< major-version 5))
(throw (ex-info "Date arithmetic not supported in versions before 5"
{:database-version mongo-version})))))
@@ -692,22 +936,25 @@ function(bin) {
(defn- interval? [expr]
(and (vector? expr) (= (first expr) :interval)))
-(defn- summarize-interval [op date-expr [_ amount unit]]
+(defn- summarize-interval [op date-expr [_ _opts amount unit]]
{op {:startDate date-expr
:unit unit
:amount amount}})
-(defn- summarize-num-or-interval [number-op date-op mongo-expr mbql-expr]
+(defn- summarize-num-or-interval
+ [query stage-number number-op date-op mongo-expr mbql-expr]
(cond
- (interval? mbql-expr) (summarize-interval date-op mongo-expr mbql-expr)
- (contains? mongo-expr number-op) (update mongo-expr number-op conj (->rvalue mbql-expr))
- :else {number-op [mongo-expr (->rvalue mbql-expr)]}))
+ (interval? mbql-expr) (summarize-interval date-op mongo-expr mbql-expr)
+ (contains? mongo-expr number-op) (update mongo-expr number-op conj (->rvalue query stage-number mbql-expr))
+ :else {number-op [mongo-expr (->rvalue query stage-number mbql-expr)]}))
-(def ^:private num-or-interval-reducer
- {:+ (partial summarize-num-or-interval "$add" "$dateAdd")
- :- (partial summarize-num-or-interval "$subtract" "$dateSubtract")})
+(defn- num-or-interval-reducer [query stage-number plus-or-minus]
+ (case plus-or-minus
+ :+ #(summarize-num-or-interval query stage-number "$add" "$dateAdd" %1 %2)
+ :- #(summarize-num-or-interval query stage-number "$subtract" "$dateSubtract" %1 %2)))
-(defmethod ->rvalue :+ [[_ & args]]
+(mu/defmethod ->rvalue :+
+ [query stage-number [_ _opts & args] :- :mbql.clause/+]
;; Addition is commutative and any but not all elements of `args` can be intervals.
;; We pick the first arg that is not an interval and add the rest of args to it.
;; (It's the callers responsibility to make sure that the first non-interval argument
@@ -716,46 +963,57 @@ function(bin) {
(if (some interval? args)
(if-let [[arg others] (u/pick-first (complement interval?) args)]
(do
- (check-date-operations-supported)
- (reduce (num-or-interval-reducer :+) (->rvalue arg) others))
+ (check-date-operations-supported query)
+ (reduce (num-or-interval-reducer query stage-number :+) (->rvalue query stage-number arg) others))
(throw (ex-info "Summing intervals is not supported" {:args args})))
- {"$add" (mapv ->rvalue args)}))
+ {"$add" (mapv (partial ->rvalue query stage-number) args)}))
-(defmethod ->rvalue :- [[_ & [arg & others :as args]]]
+(mu/defmethod ->rvalue :- :- :any ; [[mu/defmethod]] seems to get tripped up if the dispatch value is `:-` with no return value schema
+ [query stage-number [_ _opts & [arg & others :as args]] :- :mbql.clause/-]
;; Subtraction is not commutative so `arg` cannot be an interval.
;; If none of the args is an interval, we shortcut with a simple subtraction.
(if (some interval? others)
(do
- (check-date-operations-supported)
- (reduce (num-or-interval-reducer :-) (->rvalue arg) others))
- {"$subtract" (mapv ->rvalue args)}))
+ (check-date-operations-supported query)
+ (reduce (num-or-interval-reducer query stage-number :-) (->rvalue query stage-number arg) others))
+ {"$subtract" (mapv (partial ->rvalue query stage-number) args)}))
-(defmethod ->rvalue :* [[_ & args]] {"$multiply" (mapv ->rvalue args)})
+(mu/defmethod ->rvalue :*
+ [query stage-number [_ _opts & args] :- :mbql.clause/*]
+ {"$multiply" (mapv (partial ->rvalue query stage-number) args)})
-(defmethod ->rvalue :coalesce [[_ & args]] {"$ifNull" (mapv ->rvalue args)})
+(mu/defmethod ->rvalue :coalesce
+ [query stage-number [_ _opts & args] :- :mbql.clause/coalesce]
+ {"$ifNull" (mapv (partial ->rvalue query stage-number) args)})
-(defmethod ->rvalue :now [[_]]
- (if (driver/database-supports? :mongo :now (driver-api/database (driver-api/metadata-provider)))
+(mu/defmethod ->rvalue :now
+ [query _stage-number _ :- :mbql.clause/now]
+ (if (driver/database-supports? :mongo :now (driver-api/database query))
"$$NOW"
(throw (ex-info (tru "now is not supported for MongoDB versions before 4.2")
- {:database-version (:version (get-mongo-version))}))))
+ {:database-version (:version (get-mongo-version query))}))))
-(defmethod ->rvalue :text [[_ expr]]
- {"$toString" (->rvalue expr)})
+(mu/defmethod ->rvalue :text
+ [query stage-number [_ _opts expr] :- :mbql.clause/text]
+ {"$toString" (->rvalue query stage-number expr)})
-(defmethod ->rvalue :date [[_ expr]]
- (let [rvalue (->rvalue expr)]
+(mu/defmethod ->rvalue :date
+ [query stage-number [_ _opts expr] :- :mbql.clause/date]
+ (let [rvalue (->rvalue query stage-number expr)]
(with-rvalue-temporal-bucketing
+ query
{"$cond" [{"$eq" [{"$type" rvalue} "string"]}
{"$toDate" rvalue}
rvalue]}
:day)))
-(defmethod ->rvalue :today [[_]]
- (->rvalue [:date [:now]]))
+(mu/defmethod ->rvalue :today
+ [query stage-number _ :- :mbql.clause/today]
+ (->rvalue query stage-number (lib/date (lib/now))))
-(defmethod ->rvalue :datetime [[_ expr {:keys [mode]}]]
- (let [rvalue (->rvalue expr)]
+(mu/defmethod ->rvalue :datetime
+ [query stage-number [_ {:keys [mode], :as _opts} expr] :- :mbql.clause/datetime]
+ (let [rvalue (->rvalue query stage-number expr)]
(case (or mode :iso)
:iso
{"$dateFromString" {:dateString rvalue
@@ -797,16 +1055,17 @@ function(bin) {
(throw (ex-info (tru "Driver {0} does not support {1}" :mongo mode)
{:type driver-api/qp.error-type.unsupported-feature})))))
-(defmethod ->rvalue :datetime-add [[_ inp amount unit]]
- (check-date-operations-supported)
- {"$dateAdd" {:startDate (->rvalue inp)
+(mu/defmethod ->rvalue :datetime-add
+ [query stage-number [_ _opts inp amount unit] :- :mbql.clause/datetime-add]
+ (check-date-operations-supported query)
+ {"$dateAdd" {:startDate (->rvalue query stage-number inp)
:unit unit
:amount amount}})
-(defmethod ->rvalue :datetime-subtract
- [[_ inp amount unit]]
- (check-date-operations-supported)
- {"$dateSubtract" {:startDate (->rvalue inp)
+(mu/defmethod ->rvalue :datetime-subtract
+ [query stage-number [_ _opts inp amount unit]]
+ (check-date-operations-supported query)
+ {"$dateSubtract" {:startDate (->rvalue query stage-number inp)
:unit unit
:amount amount}})
@@ -855,9 +1114,10 @@ function(bin) {
{$divide [{"$dateDiff" {:startDate x, :endDate y, :unit "millisecond"}}
3600000]})
-(defmethod ->rvalue :datetime-diff [[_ x y unit]]
- (check-date-operations-supported)
- (datetime-diff (->rvalue x) (->rvalue y) unit))
+(mu/defmethod ->rvalue :datetime-diff
+ [query stage-number [_ _opts x y unit] :- :mbql.clause/datetime-diff]
+ (check-date-operations-supported query)
+ (datetime-diff (->rvalue query stage-number x) (->rvalue query stage-number y) unit))
;;; +----------------------------------------------------------------------------------------------------------------+
;;; | CLAUSE APPLICATION |
@@ -865,43 +1125,80 @@ function(bin) {
;;; ----------------------------------------------------- filter -----------------------------------------------------
-(defmethod ->rvalue ::not [[_ value]]
- {$not (->rvalue value)})
+;; represents a regex pattern that should be compiled to `{$not <regex}`
+(mr/def ::not-regex.pattern
+ [:or
+ :string
+ (lib.schema.common/instance-of-class java.util.regex.Pattern)
+ [:ref :mbql.clause/value]])
+
+(lib.schema.mbql-clause/define-mbql-clause ::not-regex :- :type/Text
+ [:tuple
+ [:= ::not-regex]
+ [:ref ::lib.schema.common/options]
+ [:ref ::not-regex.pattern]])
+
+;; TODO (Cam 2026-07-23) this actually gets compiled by [[str-match-pattern]], consider whether that behavior should
+;; get rolled into this method or we should just remove this method altogether since it's not meant to actually be
+;; used
+(mu/defmethod ->rvalue ::not-regex
+ [query stage-number [_ _opts pattern] :- ::not-regex]
+ {$not (->rvalue query stage-number pattern)})
+
+(lib.schema.mbql-clause/define-mbql-clause ::size :- :type/Integer
+ [:tuple
+ [:= ::size]
+ [:ref ::lib.schema.common/options]
+ :string])
+
+(mu/defmethod ->rvalue ::size
+ [_query _stage-number [_ _opts x]]
+ {$size x})
(defmulti compile-filter
"Compile an mbql filter clause to datastructures suitable to query mongo. Note this is not the whole query but just
compiling the \"where\" clause equivalent."
- {:added "0.39.0" :arglists '([clause])}
- driver-api/dispatch-by-clause-name-or-class)
+ {:added "0.39.0" :arglists '([query stage-number clause])}
+ (fn [_query _stage-number clause]
+ (driver-api/dispatch-by-clause-name-or-class clause)))
(defmethod compile-filter :between
- [[_ field min-val max-val]]
- (compile-filter [:and
- [:>= field min-val]
- [:<= field max-val]]))
+ [query stage-number [_ _opts expr min-val max-val]]
+ (compile-filter query stage-number (lib/and
+ (lib/>= expr min-val)
+ (lib/<= expr max-val))))
(defn- escape-regex-literal [s]
(str/replace (str s) #"[.*+?^${}()|\[\]\\]" "\\\\$0"))
-(defn- str-match-pattern [field options prefix value suffix]
- (if (driver-api/is-clause? ::not value)
- {$not (str-match-pattern field options prefix (second value) suffix)}
+(defn- str-match-pattern [query stage-number field options prefix value suffix]
+ (if (driver-api/is-clause? ::not-regex value)
+ (let [[_tag _opts pattern] value]
+ {$not (str-match-pattern query stage-number field options prefix pattern suffix)})
(do
(assert (and (contains? #{nil "^"} prefix) (contains? #{nil "$"} suffix))
"Wrong prefix or suffix value.")
- {$regexMatch {"input" (->rvalue field)
+ {$regexMatch {"input" (->rvalue query stage-number field)
"regex" (if (= (first value) :value)
- (str prefix (escape-regex-literal (->rvalue value)) suffix)
+ (str prefix (escape-regex-literal (->rvalue query stage-number value)) suffix)
{$concat (into [] (remove nil?) [(when (some? prefix) {$literal prefix})
- (->rvalue value)
+ (->rvalue query stage-number value)
(when (some? suffix) {$literal suffix})])})
"options" (if (get options :case-sensitive true) "" "i")}})))
;; these are changed to {field {$regex "regex"}} instead of {field #regex} for serialization purposes. When doing
;; native query substitution we need a string and the explicit regex form is better there
-(defmethod compile-filter :contains [[_ field v opts]] {$expr (str-match-pattern field opts nil v nil)})
-(defmethod compile-filter :starts-with [[_ field v opts]] {$expr (str-match-pattern field opts "^" v nil)})
-(defmethod compile-filter :ends-with [[_ field v opts]] {$expr (str-match-pattern field opts nil v "$")})
+(defmethod compile-filter :contains
+ [query stage-number [_ opts field v]]
+ {$expr (str-match-pattern query stage-number field opts nil v nil)})
+
+(defmethod compile-filter :starts-with
+ [query stage-number [_ opts field v]]
+ {$expr (str-match-pattern query stage-number field opts "^" v nil)})
+
+(defmethod compile-filter :ends-with
+ [query stage-number [_ opts field v]]
+ {$expr (str-match-pattern query stage-number field opts nil v "$")})
(defn- rvalue-is-variable? [rvalue]
(and (string? rvalue)
@@ -926,9 +1223,9 @@ function(bin) {
(not (rvalue-is-variable? rvalue))
(not (instance? java.util.regex.Pattern rvalue)))))
-(defn- filter-expr [operator field value]
- (let [field-rvalue (->rvalue field)
- value-rvalue (->rvalue value)]
+(defn- filter-expr [query stage-number operator field value]
+ (let [field-rvalue (->rvalue query stage-number field)
+ value-rvalue (->rvalue query stage-number value)]
(if (and (rvalue-is-field? field-rvalue)
(not (rvalue-is-field? value-rvalue))
(rvalue-can-be-compared-directly? value-rvalue))
@@ -945,199 +1242,309 @@ function(bin) {
;; {$expr {$lte [{$add [$field 1]} 100]}}
{$expr {operator [field-rvalue value-rvalue]}})))
-(defmethod compile-filter := [[_ field value]] (filter-expr $eq field value))
-(defmethod compile-filter :!= [[_ field value]] (filter-expr $ne field value))
-(defmethod compile-filter :< [[_ field value]] (filter-expr $lt field value))
-(defmethod compile-filter :> [[_ field value]] (filter-expr $gt field value))
-(defmethod compile-filter :<= [[_ field value]] (filter-expr $lte field value))
-(defmethod compile-filter :>= [[_ field value]] (filter-expr $gte field value))
+(defmethod compile-filter :=
+ [query stage-number [_ _opts field value]]
+ (filter-expr query stage-number $eq field value))
-(defmethod compile-filter :and
- [[_ & args]]
- {$and (mapv compile-filter args)})
+(defmethod compile-filter :!=
+ [query stage-number [_ _opts field value]]
+ (filter-expr query stage-number $ne field value))
-(defmethod compile-filter :or
- [[_ & args]]
- {$or (mapv compile-filter args)})
+(defmethod compile-filter :<
+ [query stage-number [_ _opts field value]]
+ (filter-expr query stage-number $lt field value))
-;; MongoDB doesn't support negating top-level filter clauses. So we can leverage the MBQL lib's `negate-filter-clause`
-;; to negate everything, with the exception of the string filter clauses, which we will convert to a `{not <regex}`
-;; clause (see `->rvalue` for `::not` above). `negate` below wraps the MBQL lib function
-(defmulti ^:private negate
- {:arglists '([mbql-clause])}
- driver-api/dispatch-by-clause-name-or-class)
+(defmethod compile-filter :>
+ [query stage-number [_ _opts field value]]
+ (filter-expr query stage-number $gt field value))
-(defmethod negate :default [clause]
- (-> clause
- lib/->mbql5
- lib/negate-boolean-expression
- lib/->legacy-MBQL))
+(defmethod compile-filter :<=
+ [query stage-number [_ _opts field value]]
+ (filter-expr query stage-number $lte field value))
-(defmethod negate :and [[_ & subclauses]] (apply vector :or (map negate subclauses)))
-(defmethod negate :or [[_ & subclauses]] (apply vector :and (map negate subclauses)))
+(defmethod compile-filter :>=
+ [query stage-number [_ _opts field value]]
+ (filter-expr query stage-number $gte field value))
-(defmethod negate :contains [[_ field v opts]] [:contains field [::not v] opts])
-(defmethod negate :starts-with [[_ field v opts]] [:starts-with field [::not v] opts])
-(defmethod negate :ends-with [[_ field v opts]] [:ends-with field [::not v] opts])
-
-(defmethod compile-filter :not [[_ subclause]]
- (compile-filter (negate subclause)))
+(defmethod compile-filter :and
+ [query stage-number [_and _opts & args]]
+ {$and (mapv (partial compile-filter query stage-number) args)})
-(defmethod compile-filter :expression [[_ expression-name]]
- (let [expression-value (driver-api/expression-with-name (:query *query*) expression-name)]
- (compile-filter expression-value)))
+(defmethod compile-filter :or
+ [query stage-number [_or _opts & args]]
+ {$or (mapv (partial compile-filter query stage-number) args)})
-(defmethod compile-filter :field [field-clause]
- {$expr {$toBool (->rvalue field-clause)}})
+;;; MongoDB doesn't support negating top-level filter clauses. So we can leverage the MBQL lib's
+;;; `negate-filter-clause` to negate everything, with the exception of the string filter clauses, which we will
+;;; convert to a `{not <regex}` clause (see `->rvalue` for `::not-regex` above). `negate` below wraps the MBQL lib
+;;; function
-(defmethod compile-filter :value [value-clause]
- {$expr (->rvalue value-clause)})
+(defmulti ^:private negate
+ {:arglists '([mbql-clause])}
+ driver-api/dispatch-by-clause-name-or-class)
-(defn- handle-filter [{filter-clause :filter} pipeline-ctx]
- (if-not filter-clause
- pipeline-ctx
- (update pipeline-ctx :query conj {$match (compile-filter filter-clause)})))
+(mu/defmethod negate :default :- ::lib.schema.mbql-clause/clause
+ [expr :- ::lib.schema.expression/boolean]
+ (lib/negate-boolean-expression expr))
+
+(mu/defmethod negate :and :- ::lib.schema.mbql-clause/clause
+ [[_ opts & subclauses] :- :mbql.clause/and]
+ (into [:or opts] (map negate) subclauses))
+
+(mu/defmethod negate :or :- ::lib.schema.mbql-clause/clause
+ [[_ opts & subclauses] :- :mbql.clause/or]
+ (into [:and opts] (map negate) subclauses))
+
+(mu/defn- not-regex :- ::not-regex
+ [pattern :- ::not-regex.pattern]
+ [::not-regex {:lib/uuid (str (random-uuid))} pattern])
+
+(mu/defmethod negate :contains :- ::lib.schema.mbql-clause/clause
+ [[_ opts field v] :- :mbql.clause/contains]
+ [:contains opts field (not-regex v)])
+
+(mu/defmethod negate :starts-with :- ::lib.schema.mbql-clause/clause
+ [[_ opts field v] :- :mbql.clause/starts-with]
+ [:starts-with opts field (not-regex v)])
+
+(mu/defmethod negate :ends-with :- ::lib.schema.mbql-clause/clause
+ [[_ opts field v] :- :mbql.clause/ends-with]
+ [:ends-with opts field (not-regex v)])
+
+(mu/defmethod compile-filter :not
+ [query stage-number [_ _opts subclause] :- :mbql.clause/not]
+ (compile-filter query stage-number (negate subclause)))
+
+(mu/defmethod compile-filter :expression
+ [query stage-number [_ _opts expression-name] :- :mbql.clause/expression]
+ (let [expression-value (lib/resolve-expression query stage-number expression-name)]
+ (compile-filter query stage-number expression-value)))
+
+(mu/defmethod compile-filter :field
+ [query stage-number field-clause :- :mbql.clause/field]
+ {$expr {$toBool (->rvalue query stage-number field-clause)}})
+
+(mu/defmethod compile-filter :value
+ [query stage-number value-clause :- :mbql.clause/value]
+ {$expr (->rvalue query stage-number value-clause)})
+
+(mu/defn- handle-filters :- ::compiled-pipeline
+ ([query stage-number pipeline-ctx]
+ (handle-filters query stage-number pipeline-ctx (lib/filters query stage-number)))
+ ([query :- ::lib.schema/query
+ stage-number :- :int
+ pipeline-ctx :- ::compiled-pipeline
+ filter-clauses :- [:maybe [:ref ::lib.schema/filters]]]
+ (if (empty? filter-clauses)
+ pipeline-ctx
+ (let [combined-filter-clause (if (> (count filter-clauses) 1)
+ (apply lib/and filter-clauses)
+ (first filter-clauses))]
+ (update pipeline-ctx :query conj {$match (compile-filter query stage-number combined-filter-clause)})))))
(defmulti ^:private compile-cond
- {:arglists '([mbql-clause])}
- driver-api/dispatch-by-clause-name-or-class)
+ {:arglists '([query stage-number mbql-clause])}
+ (mu/fn [_query _stage-number mbql-clause :- ::lib.schema.mbql-clause/clause]
+ (driver-api/dispatch-by-clause-name-or-class mbql-clause)))
-(defmethod compile-cond :between [[_ field min-val max-val]]
- (compile-cond [:and [:>= field min-val] [:<= field max-val]]))
+(mu/defmethod compile-cond :between
+ [query stage-number [_ _opts field min-val max-val] :- :mbql.clause/between]
+ (compile-cond query stage-number (lib/and (lib/>= field min-val) (lib/<= field max-val))))
(defn- index-of-code-point
"See https://docs.mongodb.com/manual/reference/operator/aggregation/indexOfCP/"
- [source needle case-sensitive?]
+ [query stage-number source needle case-sensitive?]
(let [source (if case-sensitive?
- (->rvalue source)
- {$toLower (->rvalue source)})
+ (->rvalue query stage-number source)
+ {$toLower (->rvalue query stage-number source)})
needle (if case-sensitive?
- (->rvalue needle)
- {$toLower (->rvalue needle)})]
+ (->rvalue query stage-number needle)
+ {$toLower (->rvalue query stage-number needle)})]
{:$indexOfCP [source needle]}))
-(defmethod compile-cond :contains
- [[_ field value opts]]
- {$ne [(index-of-code-point field value (get opts :case-sensitive true)) -1]})
+(mu/defmethod compile-cond :contains
+ [query stage-number [_ opts field value] :- :mbql.clause/contains]
+ {$ne [(index-of-code-point query stage-number field value (get opts :case-sensitive true)) -1]})
-(defmethod compile-cond :starts-with
- [[_ field value opts]]
- {$eq [(index-of-code-point field value (get opts :case-sensitive true)) 0]})
+(mu/defmethod compile-cond :starts-with
+ [query stage-number [_ opts field value] :- :mbql.clause/starts-with]
+ {$eq [(index-of-code-point query stage-number field value (get opts :case-sensitive true)) 0]})
-(defmethod compile-cond :ends-with
- [[_ field value opts]]
+(mu/defmethod compile-cond :ends-with
+ [query stage-number [_ opts field value] :- :mbql.clause/ends-with]
(let [strcmp (fn [a b]
{$eq (if (get opts :case-sensitive true)
[a b]
[{$strcasecmp [a b]} 0])})]
- (strcmp {:$substrCP [(->rvalue field)
- {$subtract [{:$strLenCP (->rvalue field)}
- {:$strLenCP (->rvalue value)}]}
- {:$strLenCP (->rvalue value)}]}
- (->rvalue value))))
+ (strcmp {:$substrCP [(->rvalue query stage-number field)
+ {$subtract [{:$strLenCP (->rvalue query stage-number field)}
+ {:$strLenCP (->rvalue query stage-number value)}]}
+ {:$strLenCP (->rvalue query stage-number value)}]}
+ (->rvalue query stage-number value))))
+
+(mu/defmethod compile-cond :=
+ [query stage-number [_ _opts field value] :- :mbql.clause/=]
+ {$eq [(->rvalue query stage-number field) (->rvalue query stage-number value)]})
-(defmethod compile-cond := [[_ field value]] {$eq [(->rvalue field) (->rvalue value)]})
-(defmethod compile-cond :!= [[_ field value]] {$ne [(->rvalue field) (->rvalue value)]})
-(defmethod compile-cond :< [[_ field value]] {$lt [(->rvalue field) (->rvalue value)]})
-(defmethod compile-cond :> [[_ field value]] {$gt [(->rvalue field) (->rvalue value)]})
-(defmethod compile-cond :<= [[_ field value]] {$lte [(->rvalue field) (->rvalue value)]})
-(defmethod compile-cond :>= [[_ field value]] {$gte [(->rvalue field) (->rvalue value)]})
+(mu/defmethod compile-cond :!=
+ [query stage-number [_ _opts field value] :- :mbql.clause/!=]
+ {$ne [(->rvalue query stage-number field) (->rvalue query stage-number value)]})
-(defmethod compile-cond :and [[_ & args]] {$and (mapv compile-cond args)})
-(defmethod compile-cond :or [[_ & args]] {$or (mapv compile-cond args)})
+(mu/defmethod compile-cond :<
+ [query stage-number [_ _opts field value] :- :mbql.clause/<]
+ {$lt [(->rvalue query stage-number field) (->rvalue query stage-number value)]})
-(defmethod compile-cond :not [[_ subclause]]
- (compile-cond (negate subclause)))
+(mu/defmethod compile-cond :>
+ [query stage-number [_ _opts field value] :- :mbql.clause/>]
+ {$gt [(->rvalue query stage-number field) (->rvalue query stage-number value)]})
-(defmethod compile-cond :expression [[_ expression-name]]
- (let [expression-value (driver-api/expression-with-name (:query *query*) expression-name)]
- (compile-cond expression-value)))
+(mu/defmethod compile-cond :<=
+ [query stage-number [_ _opts field value] :- :mbql.clause/<=]
+ {$lte [(->rvalue query stage-number field) (->rvalue query stage-number value)]})
-(defmethod compile-cond :field [field-clause]
- (->rvalue field-clause))
+(mu/defmethod compile-cond :>=
+ [query stage-number [_ _opts field value] :- :mbql.clause/>=]
+ {$gte [(->rvalue query stage-number field) (->rvalue query stage-number value)]})
-(defmethod compile-cond :value [value-clause]
- (->rvalue value-clause))
+(mu/defmethod compile-cond :and
+ [query stage-number [_ _opts & args] :- :mbql.clause/and]
+ {$and (mapv (partial compile-cond query stage-number) args)})
+
+(mu/defmethod compile-cond :or
+ [query stage-number [_ _opts & args] :- :mbql.clause/or]
+ {$or (mapv (partial compile-cond query stage-number) args)})
+
+(mu/defmethod compile-cond :not
+ [query stage-number [_ _opts subclause] :- :mbql.clause/not]
+ (compile-cond query stage-number (negate subclause)))
+
+(mu/defmethod compile-cond :expression
+ [query stage-number [_ _opts expression-name] :- :mbql.clause/expression]
+ (let [expression-value (lib/resolve-expression query stage-number expression-name)]
+ (compile-cond query stage-number expression-value)))
+
+(mu/defmethod compile-cond :field
+ [query stage-number field-clause :- :mbql.clause/field]
+ (->rvalue query stage-number field-clause))
+
+(mu/defmethod compile-cond :value
+ [query stage-number value-clause :- :mbql.clause/value]
+ (->rvalue query stage-number value-clause))
;;; ----------------------------------------------------- joins ------------------------------------------------------
-(defn- find-source-collection
+(mu/defn- find-source-collection :- :string
"Determine the source collection of a :join clause by recursively searching for a :source-table or a :collection
clause in :source-query clauses."
- [join-or-query]
- (or (-> join-or-query :collection)
- (some->> join-or-query :source-table (driver-api/table (driver-api/metadata-provider)) :name)
- (some-> join-or-query :source-query recur)))
+ [metadata-providerable :- ::lib.schema.metadata/metadata-providerable
+ join-or-query :- [:or
+ ::lib.schema/query
+ ::lib.schema.join/join]]
+ ;; TODO (Cam 2026-07-24) it's REALLY unclear where the hecc `:collection` is supposed
+ ;; to live, it seems to show up all over the place.
+ (or (:collection join-or-query)
+ (let [first-stage (first (:stages join-or-query))]
+ (or (:collection first-stage)
+ (get-in first-stage [:native :collection])))
+ (when-let [table-id (case (:lib/type join-or-query)
+ :mbql/query (lib/primary-source-table-id join-or-query)
+ ;; TODO (Cam 2026-07-13) -- need a better way to do this for joins...
+ :mbql/join (get-in join-or-query [:stages 0 :source-table]))]
+ (:name (driver-api/table metadata-providerable table-id)))
+ (throw (ex-info "Failed to find source collection"
+ {:join-or-query join-or-query}))))
(defn- localize-join-alias
- "Rename :join-alias properties fields to ::join-local.
+ "Rename `:join-alias` properties in `:field` ref options to `::join-local`.
+
See [[find-mapped-field-name]] for an explanation why this is done."
- [expr alias]
+ [expr join-alias]
(match/replace expr
- [:field _ {:join-alias (a :guard (= a alias))}]
- (update &match 2 set/rename-keys {:join-alias ::join-local})))
-
-(defn- get-field-mappings [source-query projections]
- (when source-query
- (zipmap (mapcat #(% source-query) [:fields :breakout :aggregation])
+ [:field {:join-alias (a :guard (= a join-alias))} _id-or-name]
+ (lib/update-options &match set/rename-keys {:join-alias ::join-local})))
+
+(mu/defn- get-field-mappings :- [:map-of ::lib.schema.mbql-clause/clause ::projection]
+ [query :- ::lib.schema/query
+ stage-number :- :int
+ projections :- [:maybe ::projections]]
+ (let [stage (lib/query-stage query stage-number)]
+ (zipmap (mapcat stage [:fields :breakout :aggregation])
projections)))
(declare ^:private mbql->native-rec)
-(defn- compile-join-source
- "Compile `source-query`, the source of a join clause, if any. Returns a map with the projections under the
- key :projections and the pipeline under the key :query.
- Handles both native and MBQL source queries."
- [source-query]
- (when source-query
- (if-let [native (:native source-query)]
- {:projections (:projections source-query)
- :query (:query native)}
- (binding [*query* (assoc (select-keys *query* [:database :type])
- :query source-query)]
- (mbql->native-rec source-query)))))
-
-(defn- handle-join [pipeline-ctx
- {:keys [alias condition source-query strategy] :as join}]
- (let [{:keys [projections], pipeline :query, :or {projections [], pipeline []}} (compile-join-source source-query)
+(defn- append-projections
+ "Projection names must be unique, for historic reasons the MongoDB driver does things like `col` and `col_2` instead
+ of `col` and `Join - col`."
+ [pipeline-ctx new-projections]
+ (letfn [(update-projections [existing-projections]
+ (let [existing-projections (vec existing-projections)
+ f (lib/non-truncating-unique-name-generator existing-projections)]
+ (into existing-projections (map f) new-projections)))]
+ (update pipeline-ctx :projections update-projections)))
+
+(mu/defn- handle-join
+ [query :- ::lib.schema/query
+ stage-number :- :int
+ pipeline-ctx
+ {join-alias :alias, :keys [conditions stages strategy], :as join}]
+ (let [join-query (assoc query :stages stages)
+ {:keys [projections], pipeline :query, :or {projections [], pipeline []}} (mbql->native-rec join-query)
;; Get the mappings introduced by the source query.
- source-field-mappings (get-field-mappings source-query projections)
+ source-field-mappings (-> (get-field-mappings join-query -1 projections)
+ (update-keys #(lib/update-options % assoc ::join-local join-alias)))
;; Find the fields the join condition refers to that are not coming from the joined query.
;; These have to be bound in the :let property of the $lookup stage, they cannot be referred to directly.
- own-fields (match/match-many condition
- [:field _ (opts :guard (not= (:join-alias opts) alias))] &match)
+ ;;
+ ;; TODO (Riley 2026-08-11) Existing issue but I think we should include `:expressions` here too, joining with
+ ;; a custom expression as the LHS fails with mongo but works with postgres
+ own-fields (match/match-many conditions
+ [:field (opts :guard (not= (:join-alias opts) join-alias)) _id-or-name] &match)
;; Map the own fields to a fresh alias and to its rvalue.
- mapping (map (fn [f] (let [alias (-> (format "let_%s_" (->lvalue f))
- ;; Mongo `$lookup` let variable names allow ASCII letters, digits,
- ;; underscores, and non-ASCII characters; any other ASCII character
- ;; (e.g. `~`, `.`, space, `-`, `:`) trips a parser error. Match only
- ;; disallowed ASCII characters so non-ASCII chars in source names are
- ;; preserved (#32182, #52807, #76722).
- (str/replace #"[\p{ASCII}&&[^A-Za-z0-9_]]" "_")
- (str "__" (next-alias-index)))]
- {:field f, :rvalue (->rvalue f), :alias alias}))
+ mapping (map (fn [field-ref]
+ (let [alias (-> (format "let_%s_" (->lvalue query stage-number field-ref))
+ ;; Mongo `$lookup` let variable names allow ASCII letters, digits,
+ ;; underscores, and non-ASCII characters; any other ASCII character
+ ;; (e.g. `~`, `.`, space, `-`, `:`) trips a parser error. Match only
+ ;; disallowed ASCII characters so non-ASCII chars in source names are
+ ;; preserved (#32182, #52807, #76722).
+ (str/replace #"[\p{ASCII}&&[^A-Za-z0-9_]]" "_")
+ (str "__" (next-alias-index)))]
+ {:field field-ref
+ :rvalue (->rvalue query stage-number field-ref)
+ :alias alias
+ :$alias (str \$ alias)}))
own-fields)]
;; Add the mappings from the source query and the let bindings of $lookup to the field mappings.
;; In the join pipeline the let bindings have to referenced with the prefix $$, so we add $ to the name.
(binding [*field-mappings* (merge *field-mappings*
source-field-mappings
- (into {} (map (juxt :field #(str \$ (:alias %)))) mapping))]
- (let [pipeline (cond-> pipeline
- condition (conj {$match (compile-filter (localize-join-alias condition alias))}))
- lookup-as (get-join-alias alias)
- stages [{$lookup {:from (find-source-collection join)
- :let (into {} (map (juxt :alias :rvalue)) mapping)
- :pipeline pipeline
- :as lookup-as}}
- {$unwind {:path (str \$ lookup-as)
- ;; left and inner joins are supported, the default is left join
- :preserveNullAndEmptyArrays (not= strategy :inner-join)}}]]
+ (into {} (map (juxt :field :$alias)) mapping))]
+ (let [filters (localize-join-alias conditions join-alias)
+ ;; TODO (Cam 2026-08-11) not clear whether this should be `query` + `stage-number` here or `join-query` +
+ ;; `-1` (for the last stage of the join?). Need to wrap my head around what's going on and then write a
+ ;; note here to clarify.
+ pipeline (-> (handle-filters query stage-number {:query pipeline} filters)
+ :query)
+ lookup-as (get-join-alias join-alias)
+ stages [{$lookup {:from (find-source-collection query join)
+ :let (into {} (map (juxt :alias :rvalue)) mapping)
+ :pipeline pipeline
+ :as lookup-as}}
+ {$unwind {:path (str \$ lookup-as)
+ ;; left and inner joins are supported, the default is left join
+ :preserveNullAndEmptyArrays (not= strategy :inner-join)}}]]
(-> pipeline-ctx
- (update :projections into projections)
+ (append-projections projections)
(update :query into stages))))))
-(defn- handle-joins [{:keys [joins]} pipeline-ctx]
- (reduce handle-join pipeline-ctx joins))
+(mu/defn- handle-joins :- ::compiled-pipeline
+ [query :- ::lib.schema/query
+ stage-number :- :int
+ pipeline-ctx :- ::compiled-pipeline]
+ (reduce (partial handle-join query stage-number)
+ pipeline-ctx
+ (lib/joins query stage-number)))
;;; -------------------------------------------------- aggregation ---------------------------------------------------
@@ -1145,119 +1552,145 @@ function(bin) {
"The set of operators handled by [[aggregation->rvalue]] and [[expand-aggregation]]."
#{:avg :count :count-where :distinct :max :min :share :stddev :sum :sum-where :var :cum-sum :cum-count})
-(defmethod ->rvalue :case [[_ cases options]]
+(mu/defmethod ->rvalue :case
+ [query stage-number [_ _opts cases default-value] :- :mbql.clause/case]
{:$switch {:branches (vec (for [[pred expr] cases]
- {:case (compile-cond pred)
- :then (->rvalue expr)}))
- :default (->rvalue (:default options))}})
-
-(defn- aggregation->rvalue [ag]
+ {:case (compile-cond query stage-number pred)
+ :then (->rvalue query stage-number expr)}))
+ :default (->rvalue query stage-number default-value)}})
+
+(mu/defn- aggregation->rvalue
+ [query :- ::lib.schema/query
+ stage-number :- :int
+ ag :- ::lib.schema.mbql-clause/clause]
(match/match-one ag
- [:aggregation-options ag' _]
- (&recur ag')
-
- [:count]
+ [:count _opts]
{$sum 1}
- [:count arg]
- {$sum {$cond {:if (->rvalue arg)
+ [:count _opts arg]
+ {$sum {$cond {:if (->rvalue query stage-number arg)
:then 1
:else 0}}}
;; these aggregation types can all be used in expressions as well so their implementations live above in the
;; general [[->rvalue]] implementations
[#{:avg :stddev :sum :min :max} & _]
- (->rvalue &match)
+ (->rvalue query stage-number &match)
- [:distinct arg]
- {$addToSet (->rvalue arg)}
+ [:distinct _opts arg]
+ {$addToSet (->rvalue query stage-number arg)}
- [:sum-where arg pred]
- {$sum {$cond {:if (compile-cond pred)
- :then (->rvalue arg)
+ [:sum-where _opts arg pred]
+ {$sum {$cond {:if (compile-cond query stage-number pred)
+ :then (->rvalue query stage-number arg)
:else 0}}}
- [:count-where pred]
- (&recur [:sum-where [:value 1] pred])
+ [:count-where _opts pred]
+ (&recur (lib/sum-where (lib/value 1) pred))
_
(throw
(ex-info (tru "Don''t know how to handle aggregation {0}" ag)
{:type :invalid-query, :clause ag}))))
-(defn- unwrap-named-ag [[ag-type arg :as ag]]
- (if (= ag-type :aggregation-options)
- (recur arg)
- ag))
-
-(defn- field-alias [[_tag _id-or-name opts, :as field-ref]]
+(defn- field-alias
+ [query stage-number [_tag opts _id-or-name, :as field-ref]]
(or (driver-api/qp.add.desired-alias opts)
- (->lvalue field-ref)))
-
-(mu/defn- breakouts-and-ags->projected-fields :- [:maybe [:sequential [:tuple driver-api/schema.common.non-blank-string :any]]]
+ (->lvalue query stage-number field-ref)))
+
+(mu/defn- breakouts-and-ags->projected-fields :- [:maybe
+ [:and
+ [:sequential [:tuple ::lib.schema.common/non-blank-string :any]]
+ [:fn
+ {:error/message "projected fields should have distinct names"}
+ (fn [projected-fields]
+ (apply distinct? (map first projected-fields)))]]]
"Determine field projections for MBQL breakouts and aggregations. Returns a sequence of pairs like
`[projected-field-name source]`."
- [breakout-fields aggregations]
- (concat
- (for [field-or-expr breakout-fields]
- [(field-alias field-or-expr) (format "$_id.%s" (field-alias field-or-expr))])
- (for [ag aggregations
- :let [ag-name (driver-api/aggregation-name (:query *query*) ag)]]
- [ag-name true])))
+ [query stage-number]
+ (let [breakouts (lib/breakouts query stage-number)
+ aggregations (lib/aggregations query stage-number)]
+ (concat
+ (for [field-or-expr breakouts
+ :let [field-alias (field-alias query stage-number field-or-expr)]]
+ [field-alias (format "$_id.%s" field-alias)])
+ (for [ag-ref aggregations
+ :let [ag-name (driver-api/mbql-5-aggregation-name query stage-number ag-ref)]]
+ [ag-name true]))))
(defmulti ^:private expand-aggregation
"Expand aggregations like `:share` and `:var` that can't be done as top-level aggregations in the `$group` stage
alone. See [[group-and-post-aggregations]] for more info. See also
https://www.mongodb.com/docs/manual/reference/operator/aggregation/group/#accumulator-operator for a list of what
aggregation operators are allowed inside `$group` (vs the ones that have to be done in a later stage)."
- {:arglists '([mbql-clause])}
- (comp first unwrap-named-ag))
-
-;;; * `:group` = stuff to do in the `$group` stage
-;;;
-;;; * `:post` = stuff to do in the `$addFields` stage immediately following it
-;;;
-;;; both of these are maps of LHS column name => RHS definition
-;;;
+ {:arglists '([query stage-number mbql-clause])}
+ (mu/fn [_query :- ::lib.schema/query
+ _stage-number :- :int
+ mbql-clause :- ::lib.schema.mbql-clause/clause]
+ (driver-api/dispatch-by-clause-name-or-class mbql-clause)))
+
;;; Note that this code doesn't handle expression aggregations, but that's ok because we do not support
;;; `:expression-aggregations` for Mongo DB.
-
-(defmethod expand-aggregation :share
- [[_ pred :as ag]]
+(mr/def ::expand-aggregation.lhs-column-name
+ :string)
+
+(mr/def ::expanded-aggregation.rhs-definition
+ ::no-uncompiled-mbql)
+
+(mr/def ::expanded-aggregation.lhs->rhs
+ [:map-of
+ ::expand-aggregation.lhs-column-name
+ ::expanded-aggregation.rhs-definition])
+
+(mr/def ::expanded-aggregation
+ [:map
+ {:closed true}
+ [:group {:description "A map containing the groups of aggregation expression. Done in the `$group` stage."}
+ ::expanded-aggregation.lhs->rhs]
+ [:post {:optional true
+ :description (str "a vector of maps containing the expressions referring to the fields generated by the"
+ " groups. Each map in the `:post` vector may (and usually does) refer to the fields"
+ " introduced by the preceding maps. Done in the `$addFields` stage immediately following"
+ " the `$group` stage.")}
+ [:sequential ::expanded-aggregation.lhs->rhs]]
+ ;; TODO (Cam 2026-07-16) document this! William added window function support to MongoDB but didn't document this
+ ;; stuff.
+ [:window {:optional true} ::expanded-aggregation.lhs->rhs]])
+
+(mu/defmethod expand-aggregation :share :- ::expanded-aggregation
+ [query stage-number [_ _opts pred :as ag] :- :mbql.clause/share]
(let [count-where-expr (name (gensym "$count-where-"))
count-expr (name (gensym "$count-"))
pred (if (= (first pred) :share)
- (second pred)
+ (nth pred 2)
pred)]
- {:group {(subs count-where-expr 1) (aggregation->rvalue [:count-where pred])
- (subs count-expr 1) (aggregation->rvalue [:count])}
- :post [{(driver-api/aggregation-name (:query *query*) ag) {$divide [count-where-expr count-expr]}}]}))
+ {:group {(subs count-where-expr 1) (aggregation->rvalue query stage-number (lib/count-where pred))
+ (subs count-expr 1) (aggregation->rvalue query stage-number (lib/count))}
+ :post [{(driver-api/mbql-5-aggregation-name query stage-number ag) {$divide [count-where-expr count-expr]}}]}))
;; MongoDB doesn't have a variance operator, but you calculate it by taking the square of the standard deviation.
;; However, `$pow` is not allowed in the `$group` stage. So calculate standard deviation in the
-(defmethod expand-aggregation :var
- [ag]
- (let [[_ expr] (unwrap-named-ag ag)
- stddev-expr (name (gensym "$stddev-"))]
- {:group {(subs stddev-expr 1) (aggregation->rvalue [:stddev expr])}
- :post [{(driver-api/aggregation-name (:query *query*) ag) {:$pow [stddev-expr 2]}}]}))
-
-(defmethod expand-aggregation :cum-sum
- [ag]
- (let [[_ expr] (unwrap-named-ag ag)
- sum-expr (name (gensym "$sum-"))]
- {:group {(subs sum-expr 1) (aggregation->rvalue [:sum expr])}
- :window {(driver-api/aggregation-name (:query *query*) ag) sum-expr}}))
-
-(defmethod expand-aggregation :cum-count
- [ag]
+(mu/defmethod expand-aggregation :var :- ::expanded-aggregation
+ [query stage-number [_tag _opts expr :as ag] :- :mbql.clause/var]
+ (let [stddev-expr (name (gensym "$stddev-"))]
+ {:group {(subs stddev-expr 1) (aggregation->rvalue query stage-number (lib/stddev expr))}
+ :post [{(driver-api/mbql-5-aggregation-name query stage-number ag) {:$pow [stddev-expr 2]}}]}))
+
+(mu/defmethod expand-aggregation :cum-sum :- ::expanded-aggregation
+ [query stage-number [_tag _opts expr :as ag] :- :mbql.clause/cum-sum]
+ (let [sum-expr (name (gensym "$sum-"))]
+ {:group {(subs sum-expr 1) (aggregation->rvalue query stage-number (lib/sum expr))}
+ :window {(driver-api/mbql-5-aggregation-name query stage-number ag) sum-expr}}))
+
+(mu/defmethod expand-aggregation :cum-count :- ::expanded-aggregation
+ [query stage-number ag :- :mbql.clause/cum-count]
(let [count-expr (name (gensym "$count-"))]
- {:group {(subs count-expr 1) (aggregation->rvalue [:count])}
- :window {(driver-api/aggregation-name (:query *query*) ag) count-expr}}))
+ {:group {(subs count-expr 1) (aggregation->rvalue query stage-number (lib/count))}
+ :window {(driver-api/mbql-5-aggregation-name query stage-number ag) count-expr}}))
-(defmethod expand-aggregation :default
- [ag]
- {:group {(driver-api/aggregation-name (:query *query*) ag) (aggregation->rvalue ag)}})
+(mu/defmethod expand-aggregation :default :- ::expanded-aggregation
+ [query stage-number ag :- ::lib.schema.mbql-clause/clause]
+ {:group {(driver-api/mbql-5-aggregation-name query stage-number ag) (aggregation->rvalue query stage-number ag)}})
(defn- extract-aggregations
"Extract aggregation expressions embedded in `aggr-expr` using `parent-name`
@@ -1265,77 +1698,100 @@ function(bin) {
The function returns a pair with the first element an expression like
`aggr-expr` with aggregations replaced by new names. The second element of
the pair is a map from the extracted aggregations to the new names conjoined
- on `aggregations-seen`. `:aggregation-option`s are ignored.
+ on `aggregations-seen`.
For example, given \"expression\" as `parent-name`, the expression
- [:aggregation-options [:+ [:count [:field 1144 nil]]
- [:* [:count [:field 1144 nil]]
- [:sum [:+ [:field 1142 nil] 1]]]]
- {:name \"expression\"}]
+ [:+ {:name \"expression\"}
+ [:count {} [:field {} 1144]]
+ [:* {}
+ [:count {} [:field {} 1144]]
+ [:sum {}
+ [:+ {}
+ [:field {} 1142]
+ 1]]]]
+
is mapped to
- [[:+ \"$expression~count\" [:* \"$expression~count\" \"$expression~sum\"]]
- {[:count [:field 1144 nil]] \"expression~count\"
- [:sum [:+ [:field 1142 nil] 1]] \"expression~sum\"}]"
- ([aggr-expr parent-name] (extract-aggregations aggr-expr parent-name {}))
- ([aggr-expr parent-name aggregations-seen]
+ [[:+ \"$expression~count\" [:* \"$expression~count\" \"$expression~sum\"]]
+ {[:count [:field 1144]] \"expression~count\"
+ [:sum [:+ [:field 1142] 1]] \"expression~sum\"}]"
+ ([query stage-number aggr-expr parent-name]
+ (extract-aggregations query stage-number aggr-expr parent-name {}))
+ ([query stage-number aggr-expr parent-name aggregations-seen]
(if (and (vector? aggr-expr) (seq aggr-expr))
- (let [[op & args] aggr-expr
- seen (get aggregations-seen aggr-expr)]
+ (let [[op opts & args] aggr-expr
+ seen (get aggregations-seen aggr-expr)]
(cond
seen
[(str \$ seen) aggregations-seen]
- (= :aggregation-options op)
- (extract-aggregations (first args) parent-name aggregations-seen)
-
(aggregation-op op)
(let [aliases-taken (set (vals aggregations-seen))
- aggr-name (driver-api/aggregation-name (:query *query*) aggr-expr)
+ aggr-name (driver-api/mbql-5-aggregation-name query stage-number aggr-expr)
desired-alias (str parent-name "~" aggr-name)
;; find a free alias by appending increasing integers
;; to the desired alias
- aggr-name (some (fn [suffix]
- (let [alias (str desired-alias suffix)]
- (when-not (aliases-taken alias)
- alias)))
- (cons "" (iterate inc 1)))]
+ aggr-name (some (fn [suffix]
+ (let [alias (str desired-alias suffix)]
+ (when-not (aliases-taken alias)
+ alias)))
+ (cons "" (iterate inc 1)))]
[(str \$ aggr-name) (assoc aggregations-seen aggr-expr aggr-name)])
:else
- (reduce (fn [[ges as] arg] ; codespell:ignore
- (let [[ge as] (extract-aggregations arg parent-name as)]
- [(conj ges ge) as])) ; codespell:ignore
- [[op] aggregations-seen]
+ (reduce (fn [[partial-clause ags-seen] arg] ; codespell:ignore
+ (let [[extracted-arg ags-seen'] (extract-aggregations
+ query
+ stage-number
+ arg
+ parent-name
+ ags-seen)]
+ [(conj partial-clause extracted-arg) ags-seen'])) ; codespell:ignore
+ [[op opts] aggregations-seen]
args)))
[aggr-expr aggregations-seen])))
(defn- simplify-extracted-aggregations
"Simplifies the extracted aggregation for `aggr-name` if the expression
contains only a single top-level aggregation. In this case there is no
- need for namespacing and `aggr-name` can be used as the name of the group
+ need for namespacing and `ag-name` can be used as the name of the group
introduced for the aggregation.
`extracted-aggr` is typically the result of [[extract-aggregations]]."
- [aggr-name [aggr-expr aggregations-seen :as extracted-aggr]]
- (if-let [aggr-group (and (string? aggr-expr)
- (str/starts-with? aggr-expr (str \$ aggr-name "~"))
+ [ag-name [extracted-ag-name aggregations-seen :as extracted-aggr]]
+ (if-let [aggr-group (and (string? extracted-ag-name)
+ (str/starts-with? extracted-ag-name (str \$ ag-name "~"))
(= (count aggregations-seen) 1)
- (let [[k v] (first aggregations-seen)]
- (when (= v (subs aggr-expr 1))
- k)))]
- [(str \$ aggr-name) {aggr-group aggr-name}]
+ (let [[seen-ag-expr seen-ag-name] (first aggregations-seen)]
+ (when (= seen-ag-name (subs extracted-ag-name 1)) ; remove prefix '$'
+ seen-ag-expr)))]
+ [(str \$ ag-name) {aggr-group ag-name}]
extracted-aggr))
-(defn- adjust-distinct-aggregations
+(mu/defn- adjust-distinct-aggregations :- [:tuple
+ ;; if this returns a vector, it must be an MBQL clause.
+ [:multi {:dispatch vector?}
+ [true ::lib.schema.mbql-clause/clause]
+ [false :any]]
+ :any]
"This function transforms `aggr-expr'` as in [[expand-aggregations]] so identifiers representing array that is
a set of _distinct_ values are wrapped in `{$size...}.
`aggr-expr` is expected to be a clause that is a result of [[extract-aggregations]]. For details see its docstring.
Distinct values are computed using the `$addToSet` in a `$group` stage. `$size` transforms them to actual count."
- [[aggr-expr mappings]]
- (let [distinct-keys (filter (fn [[clause]] (= :distinct clause)) (keys mappings))
+ [[aggr-expr mappings] :- [:tuple
+ #_aggr-expr
+ [:or
+ [:ref ::lib.schema.mbql-clause/clause]
+ :string] ; Mongo expr
+ #_mappings
+ [:map-of
+ #_ag-clause [:or
+ [:ref ::lib.schema.mbql-clause/clause]
+ :string]
+ #_name :string]]]
+ (let [distinct-keys (filter (fn [[tag, :as _ag-clause]] (= :distinct tag)) (keys mappings))
distinct-vals (into #{}
(comp (map #(get mappings %))
;; \$ is added to identifiers so eg. `q~count1` becomes `$q~count1`. Those values
@@ -1345,31 +1801,33 @@ function(bin) {
[(perf/postwalk (fn [x]
(if (and (string? x)
(distinct-vals x))
- {$size x}
+ ;; TODO (Cam 2026-08-11) sort of icky to be generating random UUIDs like this just to make
+ ;; Malli schema validation happy, and as Riley pointed out we construct clauses like this in a
+ ;; bunch of places in the SQL MBQL 5 QP code as well, we should write a Lib helper function
+ ;; for making clauses this way.
+ [::size {:lib/uuid (str (random-uuid))} x]
x))
aggr-expr)
mappings]))
-(defn- expand-aggregations
+(mu/defn- expand-aggregations :- ::expanded-aggregation
"Expands the aggregations in `aggr-expr` into groupings and post processing
- expressions. The return value is a map with the following keys:
- `:group` - a map containing the groups of aggregation expression,
- `:post` - a vector of maps containing the expressions referring to the
- fields generated by the groups. Each map in the `:post` vector may (and
- usually does) refer to the fields introduced by the preceding maps."
- [aggr-expr]
- (let [aggr-name (driver-api/aggregation-name (:query *query*) aggr-expr)
- [aggr-expr' aggregations-seen] (->> (extract-aggregations aggr-expr aggr-name)
+ expressions."
+ [query :- ::lib.schema/query
+ stage-number :- :int
+ aggr-expr :- ::lib.schema.mbql-clause/clause]
+ (let [aggr-name (driver-api/mbql-5-aggregation-name query stage-number aggr-expr)
+ [aggr-expr' aggregations-seen] (->> (extract-aggregations query stage-number aggr-expr aggr-name)
(simplify-extracted-aggregations aggr-name)
adjust-distinct-aggregations)
-
- raggr-expr (->rvalue aggr-expr')
- expandeds (map (fn [[aggr name]]
- (expand-aggregation [:aggregation-options aggr {:name name}]))
- aggregations-seen)]
- {:group (into {} (map :group) expandeds)
- :post (cond-> [(into {} (mapcat :post) expandeds)]
- (not= raggr-expr (str \$ aggr-name)) (conj {aggr-name raggr-expr}))
+ aggr-expr-rvalue (->rvalue query stage-number aggr-expr')
+ expandeds (map (fn [[ag-clause ag-name]]
+ (expand-aggregation query stage-number
+ (lib/update-options ag-clause assoc :name ag-name)))
+ aggregations-seen)]
+ {:group (into {} (map :group) expandeds)
+ :post (cond-> [(into {} (mapcat :post) expandeds)]
+ (not= aggr-expr-rvalue (str \$ aggr-name)) (conj {aggr-name aggr-expr-rvalue}))
:window (into {} (map :window) expandeds)}))
(defn- order-postprocessing
@@ -1384,14 +1842,15 @@ function(bin) {
(for [i (range (apply max (map count posts)))]
(into {} (map #(get % i)) posts))))
-(mu/defn- order-by->$sort :- [:map-of driver-api/schema.common.non-blank-string [:enum -1 1]]
- [order-by :- [:sequential driver-api/mbql.schema.OrderBy]]
+(mu/defn- order-by->$sort :- [:map-of ::lib.schema.common/non-blank-string [:enum -1 1]]
+ [query stage-number]
(into
(ordered-map/ordered-map)
- (for [[direction field] order-by]
- [(->lvalue field) (case direction
- :asc 1
- :desc -1)])))
+ (map (fn [[direction _opts field]]
+ [(->lvalue query stage-number field) (case direction
+ :asc 1
+ :desc -1)]))
+ (lib/order-bys query stage-number)))
(defn- window-output-clause
"Takes a pair of [output-name input-name] and generates an output clause suitable for
@@ -1400,69 +1859,89 @@ function(bin) {
{$sum input-name
"window" {"documents" ["unbounded" "current"]}})
-(defn- sort-lookup
+(mr/def ::window-id
+ "TODO (Cam 2026-07-24) determine actual types of keys + values"
+ [:maybe
+ [:map-of
+ ::lib.schema.common/non-blank-string
+ :any]])
+
+(mu/defn- sort-lookup
"Generates a lookup string for a particular field"
- [id name]
- (if (id name)
- (str "_id." name)
- name))
+ [id :- ::window-id
+ col-name :- ::lib.schema.common/non-blank-string]
+ (if (id col-name)
+ (str "_id." col-name)
+ col-name))
-(defn- window-sort
+(mu/defn- window-sort
"Converts a `$sort` body to something that can be used in a `sortBy` clause in a
`$setWindowFields` stage."
- [id pairs]
- (when-let [pair-seq (seq pairs)]
+ [id :- ::window-id
+ pairs]
+ (when (seq pairs)
(into (ordered-map/ordered-map)
(map (fn [[name dir]] [(sort-lookup id name) dir]))
- pair-seq)))
+ pairs)))
-(defn- window-sort-and-partitions
+(mu/defn- window-sort-and-partitions :- [:map
+ {:closed true}
+ [:sort-expr :any] ; TODO (Cam 2026-07-24) determine actual return types
+ [:partition-expr :any]]
"Calculates the appropriate sort and partition fields for a `$setWindowFields` stage."
- [id breakouts order-by]
- (let [finest-temporal-index
- (driver-api/finest-temporal-breakout-index breakouts 2)
-
- sort-index (or finest-temporal-index
- (dec (count breakouts)))
- sort-name (first (nth (seq id) sort-index))
- default-sort {(sort-lookup id sort-name) 1}
- user-sort (when order-by
- (binding [*field-mappings*
- (merge *field-mappings*
- (into {} (map (juxt identity field-alias)) breakouts))]
- (order-by->$sort order-by)))
- sort-expr (or
- ;; if there is only one breakout, always use the user's sort order
- (when (= (count id) 1)
- (window-sort id user-sort))
- ;; if we don't have a temporal breakout, sort by the last breakout, but
- ;; use the user's sort direction if specified
- (when-not finest-temporal-index
- (->> user-sort
- (filter #(= sort-name (first %)))
- (window-sort id)))
- default-sort)
+ [query :- ::lib.schema/query
+ stage-number :- :int
+ id :- ::window-id]
+ (let [breakouts (lib/breakouts query stage-number)
+ finest-temporal-index (driver-api/finest-temporal-breakout-index breakouts 1)
+ order-bys (lib/order-bys query stage-number)
+ sort-index (or finest-temporal-index
+ (dec (count breakouts)))
+ sort-name (first (nth (seq id) sort-index))
+ default-sort {(sort-lookup id sort-name) 1}
+ user-sort (when order-bys
+ (binding [*field-mappings*
+ (merge *field-mappings*
+ (into {}
+ (map (juxt identity
+ (partial field-alias query stage-number)))
+ breakouts))]
+ (order-by->$sort query stage-number)))
+ sort-expr (or
+ ;; if there is only one breakout, always use the user's sort order
+ (when (= (count id) 1)
+ (window-sort id user-sort))
+ ;; if we don't have a temporal breakout, sort by the last breakout, but
+ ;; use the user's sort direction if specified
+ (when-not finest-temporal-index
+ (->> user-sort
+ (filter #(= sort-name (first %)))
+ (window-sort id)))
+ default-sort)
partition-expr (into {}
(map (fn [[name]] [name (str "$_id." name)]))
(m/remove-nth sort-index id))]
- {:sort-expr sort-expr
+ {:sort-expr sort-expr
:partition-expr partition-expr}))
-(defn- window-accumulators
+(mu/defn- window-accumulators :- ::pipeline
"Takes a map of {output-name input-name ...} and generates a `$setWindowFields` stage that
produces a cumulative sum of those fields."
- [window-vals id breakouts order-by]
+ [query :- ::lib.schema/query
+ stage-number :- :int
+ window-vals :- [:map-of ::lib.schema.common/non-blank-string :any]
+ id :- ::window-id]
;; if id is empty, we don't have any breakouts and so don't need to fiddle around with $setWindowFields
(if (empty? id)
[{$addFields window-vals}]
- (let [{:keys [sort-expr partition-expr]} (window-sort-and-partitions id breakouts order-by)]
+ (let [{:keys [sort-expr partition-expr]} (window-sort-and-partitions query stage-number id)]
[{$setWindowFields
(cond-> {"sortBy" sort-expr
"output" (update-vals window-vals window-output-clause)}
(seq partition-expr) (assoc "partitionBy" partition-expr))}])))
-(defn- group-and-post-aggregations
+(mu/defn- group-and-post-aggregations :- ::pipeline
"Mongo is picky about which top-level aggregations it allows with groups. Eg. even
though [:/ [:count-if ...] [:count]] is a perfectly fine reduction, it's not allowed. Therefore
more complex aggregations are split in two: the reductions are done in `$group` stage after which
@@ -1473,66 +1952,72 @@ function(bin) {
The intermittent results accrued in `$group` stage are discarded in the final `$project` stage.
Meanwhile, cumulative aggregations cannot be done in either a `$group` or a `$addFields` stage
and instead need their own `$setWindowFields` stage."
- [id breakouts aggregations order-by]
- (let [expanded-ags (map expand-aggregations aggregations)
- group-ags (mapcat :group expanded-ags)
- post-ags (order-postprocessing (map :post expanded-ags))
- window-values (into {} (map :window) expanded-ags)]
+ [query :- ::lib.schema/query
+ stage-number :- :int
+ id :- ::window-id]
+ (let [aggregations (lib/aggregations query stage-number)
+ expanded-ags (map (partial expand-aggregations query stage-number) aggregations)
+ group-ags (mapcat :group expanded-ags)
+ post-ags (order-postprocessing (map :post expanded-ags))
+ window-values (into {} (map :window) expanded-ags)]
(into [{$group (into (ordered-map/ordered-map "_id" id) group-ags)}]
cat
[(when (seq window-values)
- (window-accumulators window-values id breakouts order-by))
+ (window-accumulators query stage-number window-values id))
(keep (fn [p] (when (seq p) {$addFields p}))
post-ags)])))
-(defn- projection-group-map [fields]
+(mu/defn- projection-group-map
+ [query :- ::lib.schema/query
+ stage-number :- :int]
(reduce
- (fn [m field-clause]
+ (fn [m breakout]
(assoc-in
m
- (match/match-one field-clause
- [:field (id :guard (or (integer? id) (string? id))) _]
- (str/split (field-alias field-clause) #"\.")
+ (match/match-one breakout
+ [:field _opts (id :guard (or (integer? id) (string? id)))]
+ (str/split (field-alias query stage-number breakout) #"\.")
- [:expression expr-name _]
+ [:expression _opts expr-name]
[expr-name])
- (->rvalue field-clause)))
+ (->rvalue query stage-number breakout)))
(ordered-map/ordered-map)
- fields))
+ (lib/breakouts query stage-number)))
(defn- breakouts-and-ags->pipeline-stages
"Return a sequeunce of aggregation pipeline stages needed to implement MBQL breakouts and aggregations."
- [projected-fields breakout-fields aggregations order-by]
+ [query stage-number projected-fields]
(mapcat
(partial remove nil?)
- [;; create the $group clause
+ [ ;; create the $group clause
(group-and-post-aggregations
- (when (seq breakout-fields)
- (projection-group-map breakout-fields))
- breakout-fields
- aggregations
- order-by)
- [;; Sort by _id (group)
+ query
+ stage-number
+ (when (seq (lib/breakouts query stage-number))
+ (projection-group-map query stage-number)))
+ [ ;; Sort by _id (group)
{$sort {"_id" 1}}
;; now project back to the fields we expect
{$project (into
(ordered-map/ordered-map "_id" false)
projected-fields)}]]))
-(defn- handle-breakout+aggregation
+(mu/defn- handle-breakout+aggregation :- ::compiled-pipeline
"Add projections, groupings, sortings, and other things needed to the Query pipeline context (`pipeline-ctx`) for
MBQL `aggregations` and `breakout-fields`."
- [{breakout-fields :breakout, aggregations :aggregation, :keys [order-by]} pipeline-ctx]
- (if-not (or (seq aggregations) (seq breakout-fields))
+ [query :- ::lib.schema/query
+ stage-number :- :int
+ pipeline-ctx :- ::compiled-pipeline]
+ (if-not (or (seq (lib/breakouts query stage-number))
+ (seq (lib/aggregations query stage-number)))
;; if both aggregations and breakouts are empty, there's nothing to do...
pipeline-ctx
;; determine the projections we'll need. projected-fields is like [[projected-field-name source]]`
- (let [projected-fields (breakouts-and-ags->projected-fields breakout-fields aggregations)
- pipeline-stages (breakouts-and-ags->pipeline-stages projected-fields breakout-fields aggregations order-by)]
+ (let [projected-fields (breakouts-and-ags->projected-fields query stage-number)
+ pipeline-stages (breakouts-and-ags->pipeline-stages query stage-number projected-fields)]
(-> pipeline-ctx
;; add :projections key which is just a sequence of the names of projections from above
- (assoc :projections (vec (for [[field] projected-fields]
- field)))
+ (assoc :projections (mapv first projected-fields))
;; now add additional clauses to the end of :query as applicable
(update :query into pipeline-stages)))))
@@ -1541,92 +2026,107 @@ function(bin) {
(defn- field-id->path
"Return the full document-path components for `field-id` as a vector of strings. Uses [[col->name-components]],
which prefers `:nfc-path` and falls back to walking `:parent-id` for fields synced before `:nfc-path` was populated."
- [field-id]
- (vec (col->name-components (driver-api/field (driver-api/metadata-provider) field-id))))
+ [metadata-providerable field-id]
+ (vec (col->name-components metadata-providerable (driver-api/field metadata-providerable field-id))))
-(defn- field-clauses->id->path
- "Build a map of `field-id-or-name -> path-vector` for all `:field` clauses in `fields`. Integer IDs are
+(mu/defn- field-clauses->id->path :- [:map-of
+ [:or ::lib.schema.id/field :string]
+ [:sequential :string]]
+ "Build a map of `field-id-or-name -> path-vector` for all `:field` clauses in `clauses`. Integer IDs are
resolved via the metadata provider; for string refs (e.g. from a wrapper stage), the path is derived from
the opts `:source-alias` populated by `add-alias-info` (and path-prepended by [[HACK-update-aliases]] for
nested fields), falling back to `id-or-name` when no source-alias is present. The path-joined string is
split on the Mongo path delimiter."
- [fields]
+ [metadata-providerable :- ::lib.schema.metadata/metadata-providerable
+ clauses :- [:sequential ::lib.schema.mbql-clause/clause]]
(into {}
- (keep (fn [[agg-type id-or-name opts]]
- (when (= agg-type :field)
+ (keep (fn [[tag opts id-or-name]]
+ (when (= tag :field)
(cond
- (integer? id-or-name)
- [id-or-name (field-id->path id-or-name)]
+ (pos-int? id-or-name)
+ [id-or-name (field-id->path metadata-providerable id-or-name)]
(string? id-or-name)
[id-or-name (raw-path->components
(get opts driver-api/qp.add.source-alias id-or-name))]))))
- fields))
+ clauses))
-(defn- remove-parent-fields
- "Removes any and all entries in `fields` that are parents of another field in `fields`. This is necessary because as
- of MongoDB 4.4, including both will result in an error (see:
+(mu/defn- remove-parent-fields :- [:sequential ::lib.schema.mbql-clause/clause]
+ "Removes any and all entries in `clauses` that are parents of another `:field` in `clauses` This is necessary because
+ as of MongoDB 4.4, including both will result in an error (see:
`https://www.mongodb.com/docs/manual/reference/operator/aggregation/project/#path-collision-errors-in-embedded-fields`).
Removing parents is useful when sorting, because leaf fields sort."
- [fields]
- (let [id->path (field-clauses->id->path fields)
+ [metadata-providerable :- ::lib.schema.metadata/metadata-providerable
+ clauses :- [:sequential ::lib.schema.mbql-clause/clause]]
+ (let [id->path (field-clauses->id->path metadata-providerable clauses)
parent-paths (into #{}
(keep (fn [path]
(when (> (count path) 1)
(vec (butlast path)))))
(vals id->path))]
- (remove (fn [[agg-type id-or-name & _]]
- (and (= agg-type :field)
+ (remove (fn [[tag _opts id-or-name]]
+ (and (= tag :field)
(contains? parent-paths (id->path id-or-name))))
- fields)))
+ clauses)))
-(defn- remove-child-fields
- "Removes any and all entries in `fields` that are children of another field in `fields`. This is necessary because as
- of MongoDB 4.4, including both will result in an error (see:
+(mu/defn- remove-child-fields :- [:sequential ::lib.schema.mbql-clause/clause]
+ "Removes any and all entries in `clauses` that are children of another `:field` in `clauses`. This is necessary
+ because as of MongoDB 4.4, including both will result in an error (see:
`https://www.mongodb.com/docs/manual/reference/operator/aggregation/project/#path-collision-errors-in-embedded-fields`).
Removing children is useful when projecting, because the return value of a mongo query is json, and so a parent
includes all of its children."
- [fields]
- (let [id->path (field-clauses->id->path fields)
+ [metadata-providerable :- ::lib.schema.metadata/metadata-providerable
+ clauses :- [:sequential ::lib.schema.mbql-clause/clause]]
+ (let [id->path (field-clauses->id->path metadata-providerable clauses)
all-paths (set (vals id->path))]
- (remove (fn [[agg-type id-or-name]]
- (when (= agg-type :field)
+ (remove (fn [[tag _opts id-or-name :as _clause]]
+ (when (= tag :field)
(let [path (id->path id-or-name)]
(and path
(> (count path) 1)
(contains? all-paths (vec (butlast path)))))))
- fields)))
-
-(defn- handle-order-by [{:keys [order-by breakout aggregation]} pipeline-ctx]
- (let [breakout-fields (set breakout)
- sort-fields (for [field (remove-parent-fields (map second order-by))
- ;; We only care about expressions and bucketing not added as breakout
- :when (and (not (contains? breakout-fields field))
- (let [dispatch-value
- (driver-api/dispatch-by-clause-name-or-class field)]
- (or (= :expression dispatch-value)
- (and (= :field dispatch-value)
- (let [[_ _ {:keys [temporal-unit]}] field]
- (and (some? temporal-unit)
- (not= temporal-unit :default)))))))]
- [(->lvalue field) (->rvalue field)])
+ clauses)))
+
+(mu/defn- handle-order-by :- ::compiled-pipeline
+ [query :- ::lib.schema/query
+ stage-number :- :int
+ pipeline-ctx :- ::compiled-pipeline]
+ (let [order-bys (lib/order-bys query stage-number)
+ breakouts (lib/breakouts query stage-number)
+ aggregations (lib/aggregations query stage-number)
+ breakout-fields (set breakouts)
+ sort-fields (for [field (remove-parent-fields query (map last order-bys))
+ ;; We only care about expressions and bucketing not added as breakout
+ :when (and (not (some #(lib.equality/= % field)
+ breakout-fields))
+ (let [dispatch-value (driver-api/dispatch-by-clause-name-or-class field)]
+ (or (= :expression dispatch-value)
+ (and (= :field dispatch-value)
+ (let [[_tag {:keys [temporal-unit], :as _opts} _id-or-name] field]
+ (and (some? temporal-unit)
+ (not= temporal-unit :default)))))))]
+ [(->lvalue query stage-number field) (->rvalue query stage-number field)])
;; We have already compiled breakout fields into the document.
- breakout-field-mappings (into {} (map (juxt identity field-alias)) breakout)
+ breakout-field-mappings (into {}
+ (map (juxt identity (partial field-alias query stage-number)))
+ breakouts)
;; We have already sorted ascending by the breakout fields so we don't have to repeat the
;; same sort.
explicit-order-by
- (when (and (seq order-by)
- (not= order-by (map (fn [field] [:asc field]) breakout)))
+ (when (and (seq order-bys)
+ (not (lib.equality/= order-bys
+ (map #(lib/order-by-clause % :asc) breakouts))))
(binding [*field-mappings* (merge *field-mappings* breakout-field-mappings)]
- (order-by->$sort order-by)))
+ (order-by->$sort query stage-number)))
cumulative-order-by
(when-let [finest-temporal-index
- (and (seq (filter (fn [[_ [agg-type]]] (#{:cum-sum :cum-count} agg-type)) aggregation))
- (driver-api/finest-temporal-breakout-index breakout 2))]
- (let [id (projection-group-map breakout)]
+ (and (seq (filter (fn [[agg-type :as _ag-clause]] (#{:cum-sum :cum-count} agg-type))
+ aggregations))
+ (driver-api/finest-temporal-breakout-index breakouts 1))]
+ (let [id (projection-group-map query stage-number)]
(as-> (keys id) lst
(m/remove-nth finest-temporal-index lst)
(concat lst [(nth (keys id) finest-temporal-index)])
@@ -1640,97 +2140,100 @@ function(bin) {
(concat explicit-order-by cumulative-order-by))})]
(cond-> pipeline-ctx
(seq sort-fields) (update :query conj
- ;; We $addFields before sorting, otherwise expressions will not be available for the sort
+ ;; We $addFields before sorting, otherwise expressions will not be available for the
+ ;; sort
{$addFields (into (ordered-map/ordered-map) sort-fields)})
combined-order-by (update :query #(conj % combined-order-by)))))
;;; ----------------------------------------------------- fields -----------------------------------------------------
-(defn- handle-fields [{:keys [fields]} pipeline-ctx]
- (if-not (seq fields)
- pipeline-ctx
- (let [new-projections (for [field (remove-child-fields fields)]
- [(field-alias field) (->rvalue field)])]
- (-> pipeline-ctx
- ;; we can't ask mongo for both a parent field and its child at the same time, because mongo will throw an
- ;; error. It's also unnecessary, because the parent includes the child. However, we need to list all fields
- ;; we think we want in :projections so that we know to look for them all once we get data back.
- (assoc :projections (map field-alias fields))
- ;; add project _id = false to keep _id from getting automatically returned unless explicitly specified
- (update :query conj {$project (into
- (ordered-map/ordered-map "_id" false)
- new-projections)})))))
+(mu/defn- handle-fields :- ::compiled-pipeline
+ [query :- ::lib.schema/query
+ stage-number :- :int
+ pipeline-ctx :- ::compiled-pipeline]
+ (let [fields (lib/fields query stage-number)]
+ (if (empty? fields)
+ pipeline-ctx
+ (let [new-projections (for [field (remove-child-fields query fields)]
+ [(field-alias query stage-number field) (->rvalue query stage-number field)])]
+ (-> pipeline-ctx
+ ;; we can't ask mongo for both a parent field and its child at the same time, because mongo will throw an
+ ;; error. It's also unnecessary, because the parent includes the child. However, we need to list all fields
+ ;; we think we want in :projections so that we know to look for them all once we get data back.
+ (assoc :projections (mapv (partial field-alias query stage-number) fields))
+ ;; add project _id = false to keep _id from getting automatically returned unless explicitly specified
+ (update :query conj {$project (into
+ (ordered-map/ordered-map "_id" false)
+ new-projections)}))))))
;;; ----------------------------------------------------- limit ------------------------------------------------------
-(defn- handle-limit [{:keys [limit]} pipeline-ctx]
- (if-not limit
- pipeline-ctx
- (update pipeline-ctx :query conj {$limit limit})))
+(mu/defn- handle-limit :- ::compiled-pipeline
+ [query :- ::lib.schema/query
+ stage-number :- :int
+ pipeline-ctx :- ::compiled-pipeline]
+ (let [limit (lib/current-limit query stage-number)]
+ (if-not limit
+ pipeline-ctx
+ (update pipeline-ctx :query conj {$limit limit}))))
;;; ------------------------------------------------------ page ------------------------------------------------------
-(defn- handle-page [{{page-num :page, items-per-page :items, :as page-clause} :page} pipeline-ctx]
- (if-not page-clause
- pipeline-ctx
- (update pipeline-ctx :query concat (filter some? [(let [offset (* items-per-page (dec page-num))]
- (when-not (zero? offset)
- {$skip offset}))
- {$limit items-per-page}]))))
+(mu/defn- handle-page :- ::compiled-pipeline
+ [query :- ::lib.schema/query
+ stage-number :- :int
+ pipeline-ctx :- ::compiled-pipeline]
+ (let [{page-num :page, items-per-page :items, :as page-clause} (lib/current-page query stage-number)]
+ (if-not page-clause
+ pipeline-ctx
+ (update pipeline-ctx :query concat (filter some? [(let [offset (* items-per-page (dec page-num))]
+ (when-not (zero? offset)
+ {$skip offset}))
+ {$limit items-per-page}])))))
;;; +----------------------------------------------------------------------------------------------------------------+
;;; | Process & Run |
;;; +----------------------------------------------------------------------------------------------------------------+
-(defn- add-aggregation-pipeline
- ([inner-query]
- (add-aggregation-pipeline inner-query {:projections [], :query []}))
- ([inner-query pipeline-ctx]
- (reduce (fn [pipeline-ctx f]
- (f inner-query pipeline-ctx))
+(mu/defn- add-aggregation-pipeline :- ::compiled-pipeline
+ "Generate the aggregation pipeline. Returns a sequence of maps representing each stage."
+ ([query stage-number]
+ (add-aggregation-pipeline query stage-number {:projections [], :query []}))
+ ([query :- ::lib.schema/query
+ stage-number :- :int
+ pipeline-ctx :- ::compiled-pipeline]
+ (reduce (mu/fn :- ::compiled-pipeline
+ [pipeline-ctx :- ::compiled-pipeline
+ f :- ifn?]
+ (f query stage-number pipeline-ctx))
pipeline-ctx
[#'handle-joins
- #'handle-filter
+ #'handle-filters
#'handle-breakout+aggregation
#'handle-order-by
#'handle-fields
#'handle-limit
#'handle-page])))
-(mu/defn- generate-aggregation-pipeline :- [:map
- [:projections Projections]
- [:query Pipeline]]
- "Generate the aggregation pipeline. Returns a sequence of maps representing each stage."
- [inner-query :- driver-api/MBQLQuery]
- (add-aggregation-pipeline inner-query))
-
-(defn- query->collection-name
+(mu/defn- query->collection-name :- [:maybe :string]
"Return `:collection` from a source query, if it exists."
- [query]
- (match/match-one query
- {:collection (collection :guard identity)}
- ;; ignore source queries inside `:joins` or `:collection` outside of a `:source-query`
- (when (and (some #{:source-query} &parents)
- (not (some #{:joins} &parents)))
- collection)))
+ [query :- ::lib.schema/query]
+ (some #(:collection (lib/query-stage query %))
+ (range 0 (lib/stage-count query))))
(defn- log-aggregation-pipeline [form]
(when-not driver-api/*disable-qp-logging*
(log/tracef "Compiled Mongo aggregation pipeline with %d stage(s)" (count form))))
-(defn simple-mbql->native
- "Compile a simple (non-nested) MBQL query."
- [query]
- (generate-aggregation-pipeline (or (:query query) query)))
-
-(defn parse-query-string
+(mu/defn parse-query-string :- ::pipeline
"Parse a serialized native query. Like a normal JSON parse, but handles BSON/MongoDB extended JSON forms."
- [^String s]
+ [^String s :- :string]
(try
- ;; Only way to parse _ejson array_ using bson library is through `BsonArray/parse`. That results in sequence
- ;; of `org.bson.BsonDocument`s. Currently `org.bson.Document` fits our needs better as it (1) implements `Map`
- ;; and (2) converts `BsonValue`s to java types.
- (mapv (fn [^org.bson.BsonValue v] (-> v .asDocument .toJson org.bson.Document/parse))
+ ;; Only way to parse _ejson array_ using bson library is through `BsonArray/parse`. That results in
+ ;; sequence of `org.bson.BsonDocument`s. Currently `org.bson.Document` fits our needs better as it (1)
+ ;; implements `Map` and (2) converts `BsonValue`s to java types.
+ (mapv (fn [^org.bson.BsonValue v]
+ (-> v .asDocument .toJson org.bson.Document/parse))
(org.bson.BsonArray/parse s))
(catch Throwable e
(throw (ex-info (tru "Unable to parse query: {0}" (.getMessage e))
@@ -1738,34 +2241,42 @@ function(bin) {
:query s}
e)))))
-(defn- mbql->native-rec
- "Compile a potentially nested MBQL query."
- [inner-query]
- (if-let [source-query (-> inner-query :source-query)]
- (let [compiled (or (when-let [nq (:native source-query)]
- (cond
- (string? (:query nq))
- (-> source-query
- (dissoc :native)
- (assoc :query (parse-query-string (:query nq))))
-
- :else
- nq))
- (binding [*nesting-level* (inc *nesting-level*)]
- (mbql->native-rec source-query)))
- field-mappings (get-field-mappings source-query (:projections compiled))]
- (binding [*field-mappings* field-mappings]
- (merge compiled (add-aggregation-pipeline inner-query compiled))))
- (simple-mbql->native inner-query)))
+(mu/defn- mbql->native-rec :- ::compiled-pipeline
+ "Compile an MBQL 5 query."
+ [query :- ::lib.schema/query]
+ (transduce
+ (map (mu/fn [stage-number :- :int]
+ (let [compiled (if (lib/native-stage? query stage-number)
+ (let [raw-native-query (lib/raw-native-query query)
+ native-query-map (if (map? raw-native-query)
+ raw-native-query
+ {:query raw-native-query
+ :projections []})]
+ (update native-query-map :query (fn [query]
+ (cond-> query
+ (string? query) parse-query-string))))
+ {:query [], :projections []})
+ field-mappings (get-field-mappings query stage-number (:projections compiled))]
+ (binding [*field-mappings* field-mappings]
+ (merge compiled (add-aggregation-pipeline query stage-number compiled))))))
+ (completing
+ (mu/fn :- ::compiled-pipeline
+ [acc :- ::compiled-pipeline compiled-stage :- ::compiled-pipeline]
+ (-> acc
+ (update :query into (:query compiled-stage))
+ (assoc :projections (:projections compiled-stage [])))))
+ {:query [], :projections []}
+ (range (lib/stage-count query))))
;;; TODO (Cam 6/20/25) -- MongoDB QP code is completely broken and does not consistently look at the keys added
;;; by [[driver-api/add-alias-info]]. Fixing all the busted code above is more work than I want to take on right now, so
;;; until we get around to fixing that let's just walk the query and replace all the non-add-alias-info keys with the
;;; values added by add-alias-info.
-(defn- HACK-update-aliases [form]
+(defn- HACK-update-aliases-in-field-refs
+ [query]
(letfn [(prepend-nfc-path [{nfc-path driver-api/qp.add.nfc-path,
- source-alias driver-api/qp.add.source-alias,
- desired-alias driver-api/qp.add.desired-alias,
+ source-alias driver-api/qp.add.source-alias
+ desired-alias driver-api/qp.add.desired-alias
:as opts}]
(when (seq nfc-path)
(let [nfc-path-str (str/join \. nfc-path)]
@@ -1798,39 +2309,73 @@ function(bin) {
update-name
remove-bad-join-alias
update-join-alias]))
- (update-field-ref [[_tag id-or-name {source-alias driver-api/qp.add.source-alias, :as opts}]]
- (let [opts (update-opts opts)]
- (if (and (string? id-or-name)
- source-alias)
- [:field source-alias opts]
- [:field id-or-name opts])))]
- (match/replace form
- [:field & _]
- (update-field-ref &match)
-
- (:and join
- {driver-api/qp.add.alias (add-alias :guard (and add-alias (not= add-alias (:alias join))))})
- (&recur (assoc join :alias add-alias)))))
-
-(defn- preprocess
- [inner-query]
- (-> inner-query
+ (update-field-ref [[_tag {source-alias driver-api/qp.add.source-alias, :as _opts} id-or-name, :as field-ref]]
+ (let [field-ref' (lib/update-options field-ref update-opts)]
+ (cond-> field-ref'
+ (and (string? id-or-name)
+ source-alias)
+ (assoc 2 source-alias))))]
+ ;; disable enforcement as the query won't be fully valid until finishing [[HACK-update-aliases-in-joins]]
+ (mu/disable-enforcement
+ (lib.walk/walk-clauses
+ query
+ (fn [_query _path-type _stage-or-join-path clause]
+ (cond-> clause
+ (and (vector? clause)
+ (= (first clause) :field))
+ update-field-ref))))))
+
+(defn- HACK-update-aliases-in-joins
+ [query]
+ ;; disable enforcement as the query won't be fully valid until finishing this step
+ (mu/disable-enforcement
+ (lib.walk/walk
+ query
+ (fn [_query path-type _path join]
+ (when (= path-type :lib.walk/join)
+ (let [add-alias (driver-api/qp.add.alias join)]
+ (when (and add-alias
+ (not= add-alias (:alias join)))
+ (letfn [(update-field [field]
+ (lib/with-join-alias field add-alias))
+ (update-fields [fields]
+ (mapv update-field fields))]
+ (-> join
+ (assoc :alias add-alias)
+ (m/update-existing :fields update-fields))))))))))
+
+(mu/defn- HACK-update-aliases :- ::lib.schema/query
+ [query :- ::lib.schema/query]
+ (-> query
+ HACK-update-aliases-in-field-refs
+ HACK-update-aliases-in-joins))
+
+(mu/defn- preprocess :- ::lib.schema/query
+ [query :- ::lib.schema/query]
+ (-> query
(driver-api/add-alias-info {:globally-unique-join-aliases? true})
HACK-update-aliases))
-(defn mbql->native
+(mr/def ::compiled-pipeline-with-collection
+ "This is the same as `::compiled-pipeline` but `:collection` is required, and `:query` can remain an unparsed
+ string."
+ [:merge
+ [:ref ::compiled-pipeline]
+ [:map
+ [:query [:or [:ref ::pipeline] :string]]
+ [:collection :string]]])
+
+(mu/defn mbql->native :- ::compiled-pipeline-with-collection
"Compile an MBQL query."
- [query]
- (let [query (-> query
- driver-api/->legacy-MBQL
- (update :query preprocess))]
- (binding [*query* query
- *next-alias-index* (volatile! 0)]
- (let [source-table-name (if-let [source-table-id (driver-api/query->source-table-id query)]
- (:name (driver-api/table (driver-api/metadata-provider) source-table-id))
+ [query :- ::lib.schema/query]
+ (let [query (preprocess query)]
+ (binding [*next-alias-index* (volatile! 0)]
+ (let [source-table-name (if-let [source-table-id (lib/primary-source-table-id query)]
+ (:name (driver-api/table query source-table-id))
(query->collection-name query))
- compiled (mbql->native-rec (:query query))]
+ compiled (mbql->native-rec query)]
(log-aggregation-pipeline (:query compiled))
(assoc compiled
- :collection source-table-name
- :mbql? true)))))
+ :collection (or source-table-name
+ (find-source-collection query query))
+ :mbql? true)))))
diff --git a/modules/drivers/mongo/test/metabase/driver/mongo/parameters_test.clj b/modules/drivers/mongo/test/metabase/driver/mongo/parameters_test.clj
index a612a7f7153..8068b14a578 100644
--- a/modules/drivers/mongo/test/metabase/driver/mongo/parameters_test.clj
+++ b/modules/drivers/mongo/test/metabase/driver/mongo/parameters_test.clj
@@ -45,78 +45,85 @@
(map? options) (assoc :options options)))
options)))
+(defn- substitute [param->value x]
+ (#'mongo.params/substitute
+ (lib/native-query meta/metadata-provider "[]")
+ 0
+ param->value
+ x))
+
(deftest ^:parallel substitute-test
(testing "non-parameterized strings should not be substituted"
(is (= "wow"
- (#'mongo.params/substitute nil ["wow"])))))
+ (substitute nil ["wow"])))))
(deftest ^:parallel substitute-test-2
(testing "non-optional-params"
(testing "single param with no string before or after"
(is (= "100"
- (#'mongo.params/substitute {"x" 100} [(lib/parsed-param "x")]))
+ (substitute {"x" 100} [(lib/parsed-param "x")]))
"\"{{x}}\" with x = 100 should be replaced with \"100\""))
(testing "if a param is missing, an Exception should be thrown"
(is (thrown-with-msg? clojure.lang.ExceptionInfo
#"missing required parameters: #\{\"x\"\}"
- (#'mongo.params/substitute nil [(lib/parsed-param "x")]))))
+ (substitute nil [(lib/parsed-param "x")]))))
(testing "params preceded or followed by strings should get combined into a single string"
(is (= "2100"
- (#'mongo.params/substitute {"x" 100} ["2" (lib/parsed-param "x")]))
+ (substitute {"x" 100} ["2" (lib/parsed-param "x")]))
"\"2{{x}}\" with x = 100 should be replaced with string \"2100\""))
(testing "temporal params"
(is (= "ISODate(\"2019-12-06T17:01:00-08:00\")"
- (#'mongo.params/substitute {"written-at" #t "2019-12-06T17:01:00-08:00[America/Los_Angeles]"}
- [(lib/parsed-param "written-at")]))))
+ (substitute {"written-at" #t "2019-12-06T17:01:00-08:00[America/Los_Angeles]"}
+ [(lib/parsed-param "written-at")]))))
(testing "multiple params in one string"
(is (= "2019-12-10"
- (#'mongo.params/substitute {"year" 2019, "month" 12, "day" 10} [(lib/parsed-param "year") "-" (lib/parsed-param "month") "-" (lib/parsed-param "day")]))
+ (substitute {"year" 2019, "month" 12, "day" 10} [(lib/parsed-param "year") "-" (lib/parsed-param "month") "-" (lib/parsed-param "day")]))
"\"{{year}}-{{month}}-{{day}}\" should be replaced with \"2019-12-09\"")
(testing "some params missing"
(is (thrown-with-msg? clojure.lang.ExceptionInfo
#"missing required parameters: #\{\"day\"\}"
- (#'mongo.params/substitute {"year" 2019, "month" 12} [(lib/parsed-param "year") "-" (lib/parsed-param "month") "-" (lib/parsed-param "day")])))))))
+ (substitute {"year" 2019, "month" 12} [(lib/parsed-param "year") "-" (lib/parsed-param "month") "-" (lib/parsed-param "day")])))))))
(deftest ^:parallel substitute-test-3
(testing "optional params"
(testing "single optional param"
(is (= nil
- (#'mongo.params/substitute nil [(optional (lib/parsed-param "x"))]))
+ (substitute nil [(optional (lib/parsed-param "x"))]))
"\"[[{{x}}]]\" with no value for x should be replaced with `nil`"))
(testing "{{year}}[[-{{month}}[[-{{day}}]]]]"
(let [params [(lib/parsed-param "year")
(optional "-" (lib/parsed-param "month") (optional "-" (lib/parsed-param "day")))]]
(testing "with all params present"
(is (= "2019-12-10"
- (#'mongo.params/substitute {"year" 2019, "month" 12, "day" 10} params))))
+ (substitute {"year" 2019, "month" 12, "day" 10} params))))
(testing "with :year & :month present but not :day"
(is (= "2019-12"
- (#'mongo.params/substitute {"year" 2019, "month" 12} params))))
+ (substitute {"year" 2019, "month" 12} params))))
(testing "with :year present but not :month or :day"
(is (= "2019"
- (#'mongo.params/substitute {"year" 2019} params))))
+ (substitute {"year" 2019} params))))
(testing "with no params present"
(is (thrown-with-msg? clojure.lang.ExceptionInfo
#"missing required parameters"
- (#'mongo.params/substitute nil params))))))))
+ (substitute nil params))))))))
(deftest ^:parallel substitute-test-4
(testing "comma-separated numbers"
(is (= "{$in: [1, 2, 3]}"
- (#'mongo.params/substitute {"id" [1 2 3]}
- [(lib/parsed-param "id")])))))
+ (substitute {"id" [1 2 3]}
+ [(lib/parsed-param "id")])))))
(deftest ^:parallel substitute-test-5
(testing "multiple-values single (#30136)"
(is (= "\"33 Taps\""
- (#'mongo.params/substitute {"id" ["33 Taps"]}
- [(lib/parsed-param "id")])))))
+ (substitute {"id" ["33 Taps"]}
+ [(lib/parsed-param "id")])))))
(deftest ^:parallel substitute-test-6
(testing "multiple-values multi (#22486)"
(is (= "{$in: [\"33 Taps\", \"Cha Cha Chicken\"]}"
- (#'mongo.params/substitute {"id" ["33 Taps" "Cha Cha Chicken"]}
- [(lib/parsed-param "id")])))))
+ (substitute {"id" ["33 Taps" "Cha Cha Chicken"]}
+ [(lib/parsed-param "id")])))))
(defprotocol ^:private ToBSON
(^:private to-bson [this]
@@ -165,8 +172,8 @@
(testing "Date ranges"
(mt/with-clock #t "2019-12-13T12:00:00.000Z[UTC]"
(letfn [(substitute-date-range [s]
- (#'mongo.params/substitute {"date" (field-filter "date" :date/range s)}
- ["[{$match: " (lib/parsed-param "date") "}]"]))]
+ (substitute {"date" (field-filter "date" :date/range s)}
+ ["[{$match: " (lib/parsed-param "date") "}]"]))]
(is (= (to-bson [{:$match {:$and [{"date" {:$gte (ISODate "2019-12-08T00:00:00Z")}}
{"date" {:$lt (ISODate "2019-12-13T00:00:00Z")}}]}}])
(substitute-date-range "past5days")))
@@ -181,24 +188,24 @@
(deftest ^:parallel field-filter-test-2
(testing "multiple values (numbers)"
(is (= (to-bson [{:$match {"id" {:$in [1 2 3]}}}])
- (#'mongo.params/substitute {"id" (field-filter "id" :number [1 2 3])}
- ["[{$match: " (lib/parsed-param "id") "}]"])))))
+ (substitute {"id" (field-filter "id" :number [1 2 3])}
+ ["[{$match: " (lib/parsed-param "id") "}]"])))))
(deftest field-filter-test-3
(mt/with-temporary-setting-values [report-timezone "UTC"]
(testing "single date"
(is (= (to-bson [{:$match {:$and [{"date" {:$gte (ISODate "2019-12-08")}}
{"date" {:$lt (ISODate "2019-12-09")}}]}}])
- (#'mongo.params/substitute {"date" (field-filter "date" :date/single "2019-12-08")}
- ["[{$match: " (lib/parsed-param "date") "}]"]))))))
+ (substitute {"date" (field-filter "date" :date/single "2019-12-08")}
+ ["[{$match: " (lib/parsed-param "date") "}]"]))))))
(deftest ^:parallel field-filter-test-4
(testing "parameter not supplied"
(is (= (to-bson [{:$match {}}])
- (#'mongo.params/substitute {"date" (lib/parsed-field-filter-param
- {:lib/type :metadata/column, :name "date", :base-type :type/Date}
- lib/parsed-param-no-value-placeholder)}
- ["[{$match: " (lib/parsed-param "date") "}]"])))))
+ (substitute {"date" (lib/parsed-field-filter-param
+ {:lib/type :metadata/column, :name "date", :base-type :type/Date}
+ lib/parsed-param-no-value-placeholder)}
+ ["[{$match: " (lib/parsed-param "date") "}]"])))))
(deftest ^:parallel field-filter-test-5
(testing "operators"
@@ -230,8 +237,8 @@
(testing operator
(is (= (strip (to-bson [{:$match form}]))
(strip
- (#'mongo.params/substitute {"desc" (field-filter "description" :type/Text operator input options)}
- ["[{$match: " (lib/parsed-param "desc") "}]"])))))))))
+ (substitute {"desc" (field-filter "description" :type/Text operator input options)}
+ ["[{$match: " (lib/parsed-param "desc") "}]"])))))))))
(deftest ^:parallel field-filter-test-6
(testing "operators"
@@ -246,8 +253,8 @@
(testing operator
(is (= (strip (to-bson [{:$match form}]))
(strip
- (#'mongo.params/substitute {"price" (field-filter "price" :type/Integer operator input)}
- ["[{$match: " (lib/parsed-param "price") "}]"])))))))))
+ (substitute {"price" (field-filter "price" :type/Integer operator input)}
+ ["[{$match: " (lib/parsed-param "price") "}]"])))))))))
(deftest ^:parallel field-filter-test-7
(testing "operators"
@@ -257,8 +264,8 @@
(is (= (strip (to-bson [{:$match {"$or" [{"description" "foo"}
{"description" "bar"}]}}]))
(strip
- (#'mongo.params/substitute {"desc" (field-filter "description" :type/Text :string/= ["foo" "bar"])}
- ["[{$match: " (lib/parsed-param "desc") "}]"])))))
+ (substitute {"desc" (field-filter "description" :type/Text :string/= ["foo" "bar"])}
+ ["[{$match: " (lib/parsed-param "desc") "}]"])))))
(testing :string/!=
;; this could be optimized to {:description {:in ["foo" "bar"}}? one thing is that we pass it through the
;; desugar middleware that does this [:= 1 2] -> [:or [:= 1] [:= 2]] which makes for more complicated (or just
@@ -267,29 +274,30 @@
(is (= (strip (to-bson [{:$match {"$and" [{"description" {"$ne" "foo"}}
{"description" {"$ne" "bar"}}]}}]))
(strip
- (#'mongo.params/substitute {"desc" (field-filter "description" :type/Text :string/!= ["foo" "bar"])}
- ["[{$match: " (lib/parsed-param "desc") "}]"])))))
+ (substitute {"desc" (field-filter "description" :type/Text :string/!= ["foo" "bar"])}
+ ["[{$match: " (lib/parsed-param "desc") "}]"])))))
(testing :number/=
(is (= (strip (to-bson [{:$match {"$or" [{"price" 1}
{"price" 2}]}}]))
(strip
- (#'mongo.params/substitute {"price" (field-filter "price" :type/Integer :number/= [1 2])}
- ["[{$match: " (lib/parsed-param "price") "}]"])))))
+ (substitute {"price" (field-filter "price" :type/Integer :number/= [1 2])}
+ ["[{$match: " (lib/parsed-param "price") "}]"])))))
(testing :number/!=
(is (= (strip (to-bson [{:$match {"$and" [{"price" {"$ne" 1}}
{"price" {"$ne" 2}}]}}]))
(strip
- (#'mongo.params/substitute {"price" (field-filter "price" :type/Integer :number/!= [1 2])}
- ["[{$match: " (lib/parsed-param "price") "}]"]))))))))
+ (substitute {"price" (field-filter "price" :type/Integer :number/!= [1 2])}
+ ["[{$match: " (lib/parsed-param "price") "}]"]))))))))
(deftest ^:parallel substitute-native-query-snippets-test
(testing "Native query snippet substitution"
(is (= (strip (to-bson [{:$match {"price" {:$gt 2}}}]))
- (strip (#'mongo.params/substitute {"snippet: high price"
- {:lib/type :metabase.lib.parameters.parse.types/referenced-query-snippet
- :snippet-id 123
- :content (to-bson {"price" {:$gt 2}})}}
- ["[{$match: " (lib/parsed-param "snippet: high price") "}]"]))))))
+ (strip (substitute
+ {"snippet: high price"
+ {:lib/type :metabase.lib.parameters.parse.types/referenced-query-snippet
+ :snippet-id 123
+ :content (to-bson {"price" {:$gt 2}})}}
+ ["[{$match: " (lib/parsed-param "snippet: high price") "}]"]))))))
(deftest ^:parallel substitute-native-parameters-test
(let [mp meta/metadata-provider
@@ -306,13 +314,13 @@
:parameters [{:type :date/range
:target [:dimension [:template-tag "date"]]
:value "2014-03-01~2014-03-02"}]})
- stage (-> (lib/query-stage query 0)
- (assoc :parameters (:parameters query)))]
+ query (lib/update-query-stage query 0 assoc :parameters (:parameters query))]
(is (=? {:native (to-bson
[{:$match {:$and [{"DATE" {:$gte (ISODate "2014-03-01T00:00:00Z")}}
{"DATE" {:$lt (ISODate "2014-03-03T00:00:00Z")}}]}}
{:$sort {:_id 1}}])}
- (mongo.params/substitute-native-parameters :mongo mp stage)))))
+ (-> (mongo.params/substitute-native-parameters :mongo query 0)
+ (lib/query-stage 0))))))
(deftest ^:parallel variable-default-value-substitution-test
(testing "previewing a Mongo native query substitutes a plain variable tag's :default when no runtime param is supplied (#47793)"
@@ -326,11 +334,11 @@
:display-name "Quantity"
:type :number
:default "10"
- :required false}}))
- stage (-> (lib/query-stage query 0)
- (assoc :parameters (or (:parameters query) [])))]
+ :required false}})
+ (as-> $query (lib/update-query-stage $query 0 assoc :parameters (or (:parameters $query) []))))]
(is (=? {:native (to-bson [{:$match {"quantity" 10}}])}
- (mongo.params/substitute-native-parameters :mongo mp stage))))))
+ (-> (mongo.params/substitute-native-parameters :mongo query 0)
+ (lib/query-stage 0)))))))
(deftest e2e-field-filter-test
(mt/test-driver :mongo
diff --git a/modules/drivers/mongo/test/metabase/driver/mongo/query_processor_test.clj b/modules/drivers/mongo/test/metabase/driver/mongo/query_processor_test.clj
index ab37834d5fa..c55feb4b68f 100644
--- a/modules/drivers/mongo/test/metabase/driver/mongo/query_processor_test.clj
+++ b/modules/drivers/mongo/test/metabase/driver/mongo/query_processor_test.clj
@@ -10,6 +10,8 @@
[metabase.driver.mongo.query-processor :as mongo.qp]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
+ [metabase.lib.schema.mbql-clause :as lib.schema.mbql-clause]
+ [metabase.lib.test-metadata :as meta]
[metabase.lib.test-util :as lib.tu]
[metabase.query-processor.alternative-date-test :as qp.alternative-date-test]
[metabase.query-processor.compile :as qp.compile]
@@ -18,27 +20,34 @@
[metabase.query-processor.test :as qp]
[metabase.query-processor.timezone :as qp.timezone]
[metabase.test :as mt]
- [metabase.util.json :as json]))
+ [metabase.util.json :as json]
+ [metabase.util.malli :as mu]))
(set! *warn-on-reflection* true)
+(defn- query->collection-name
+ [query]
+ (#'mongo.qp/query->collection-name
+ (lib/query meta/metadata-provider (merge {:database (meta/id), :type :query} query))))
+
(deftest ^:parallel query->collection-name-test
(testing "query->collection-name"
(testing "should be able to extract :collection from :source-query"
(is (= "checkins"
- (#'mongo.qp/query->collection-name {:query {:source-query
- {:collection "checkins"
- :native []}}}))))
+ (query->collection-name {:query {:source-query
+ {:collection "checkins"
+ :native []}}}))))
(testing "should work for nested-nested queries"
(is (= "checkins"
- (#'mongo.qp/query->collection-name {:query {:source-query {:source-query
- {:collection "checkins"
- :native []}}}}))))
+ (query->collection-name {:query {:source-query {:source-query
+ {:collection "checkins"
+ :native []}}}}))))
(testing "should ignore :joins"
(is (= nil
- (#'mongo.qp/query->collection-name {:query {:source-query
- {:native []}
- :joins [{:source-query "wow"}]}}))))))
+ (query->collection-name {:query {:source-query
+ {:native []}
+ :joins [{:source-query {:source-table 1, :collection "wow"}
+ :condition [:= 1 2]}]}}))))))
(deftest ^:parallel order-postprocessing-test
(is (= [{"expression_2~share" {"$divide" ["$count-where-141638" "$count-141639"]}}
@@ -51,42 +60,54 @@
[{"expression_2~share" {"$divide" ["$count-where-141638" "$count-141639"]}}
{"expression_2" {"$multiply" [2 "$expression_2~share"]}}]]))))
-(deftest ^:parallel relative-datetime-test
+(deftest relative-datetime-test
(mt/test-driver :mongo
(testing "Make sure relative datetimes are compiled sensibly"
(mt/with-clock #t "2021-02-17T10:36:00-08:00[US/Pacific]"
- (mt/dataset attempted-murders
- (is (= {:projections ["count"]
- :query [{"$match"
- {"$and"
- [{"$expr" {"$gte" ["$datetime" {:$dateFromString {:dateString "2021-01-01T00:00Z"}}]}}
- {"$expr" {"$lt" ["$datetime" {:$dateFromString {:dateString "2021-02-01T00:00Z"}}]}}]}}
- {"$group" {"_id" nil, "count" {"$sum" 1}}}
- {"$sort" {"_id" 1}}
- {"$project" {"_id" false, "count" true}}]
- :collection "attempts"
- :mbql? true}
- (qp.compile/compile
- (mt/mbql-query attempts
- {:aggregation [[:count]]
- :filter [:time-interval $datetime :last :month]})))))))))
+ (mt/with-temporary-setting-values [report-timezone "UTC"]
+ (mt/dataset attempted-murders
+ (is (= {:projections ["count"]
+ :query [{"$match"
+ {"$and"
+ [{"$expr" {"$gte" ["$datetime" {:$dateFromString {:dateString "2021-01-01T00:00Z"}}]}}
+ {"$expr" {"$lt" ["$datetime" {:$dateFromString {:dateString "2021-02-01T00:00Z"}}]}}]}}
+ {"$group" {"_id" nil, "count" {"$sum" 1}}}
+ {"$sort" {"_id" 1}}
+ {"$project" {"_id" false, "count" true}}]
+ :collection "attempts"
+ :mbql? true}
+ (qp.compile/compile
+ (mt/mbql-query attempts
+ {:aggregation [[:count]]
+ :filter [:time-interval $datetime :last :month]}))))))))))
+
+(mu/defn- compile-filter [filter-clause :- ::lib.schema.mbql-clause/clause]
+ (mongo.qp/compile-filter
+ (lib/native-query meta/metadata-provider "[]")
+ -1
+ filter-clause))
(deftest ^:parallel absolute-datetime-test
(mt/test-driver :mongo
(mt/with-metadata-provider (mt/id)
- (testing "Make sure absolute-datetime are compiled correctly"
- (doseq [[expected date]
+ (testing "Make sure :absolute-datetime and :time clauses are compiled correctly"
+ (doseq [[expected t]
[["2014-01-01" (t/local-date "2014-01-01")]
["10:00" (t/local-time "10:00:00")]
["2014-01-01T10:00" (t/local-date-time "2014-01-01T10:00")]
["03:00Z" (t/offset-time "10:00:00+07:00")]
["2014-01-01T03:00Z" (t/offset-date-time "2014-01-01T10:00+07:00")]
["2014-01-01T00:00Z" (t/zoned-date-time "2014-01-01T07:00:00+07:00[Asia/Ho_Chi_Minh]")]]]
- (testing (format "with %s" (type date))
+ (testing (format "with %s" (type t))
(is (= {"$expr" {"$lt" ["$date-field" {:$dateFromString {:dateString expected}}]}}
- (mongo.qp/compile-filter [:<
- [:field "date-field"]
- [:absolute-datetime date]])))))))))
+ (compile-filter (lib/<
+ (lib/ref {:lib/type :metadata/column
+ :base-type :type/Date
+ :name "date-field"})
+ (let [f (if ((some-fn t/local-time? t/offset-time?) t)
+ lib/time
+ lib/absolute-datetime)]
+ (f t :default))))))))))))
(defn- date-arithmetic-supported? []
(driver/database-supports? :mongo :date-arithmetics (mt/db)))
@@ -197,27 +218,27 @@
(testing "Result timezone is respected when grouping by hour (#11149)"
(mt/dataset attempted-murders
(testing "Querying in UTC works"
- (mt/with-system-timezone-id! "UTC"
+ (mt/with-temporary-setting-values [report-timezone "UTC"]
(is (= [["2019-11-20T20:00:00Z" 1]
["2019-11-19T00:00:00Z" 1]
["2019-11-18T20:00:00Z" 1]
["2019-11-17T14:00:00Z" 1]]
(mt/rows (mt/run-mbql-query attempts
{:aggregation [[:count]]
- :breakout [[:field %datetime {:temporal-unit :hour}]]
- :order-by [[:desc [:field %datetime {:temporal-unit :hour}]]]
- :limit 4}))))))
+ :breakout [[:field %datetime {:temporal-unit :hour}]]
+ :order-by [[:desc [:field %datetime {:temporal-unit :hour}]]]
+ :limit 4}))))))
(testing "Querying in Kathmandu works"
- (mt/with-system-timezone-id! "Asia/Kathmandu"
+ (mt/with-temporary-setting-values [report-timezone "Asia/Kathmandu"]
(is (= [["2019-11-21T01:00:00+05:45" 1]
["2019-11-19T06:00:00+05:45" 1]
["2019-11-19T02:00:00+05:45" 1]
["2019-11-17T19:00:00+05:45" 1]]
(mt/rows (mt/run-mbql-query attempts
{:aggregation [[:count]]
- :breakout [[:field %datetime {:temporal-unit :hour}]]
- :order-by [[:desc [:field %datetime {:temporal-unit :hour}]]]
- :limit 4}))))))))))
+ :breakout [[:field %datetime {:temporal-unit :hour}]]
+ :order-by [[:desc [:field %datetime {:temporal-unit :hour}]]]
+ :limit 4}))))))))))
(deftest ^:parallel nested-columns-test
(mt/test-driver :mongo
@@ -248,10 +269,12 @@
{:aggregation [[:count]]
:breakout [$tips.source.username]}))))
(testing "Nested fields in join condition aliases are transformed to use `_` instead of a `.` (#32182)"
- (let [query (mt/mbql-query tips
- {:joins [{:alias "Tips"
- :source-table $$tips
- :condition [:= $tips.source.categories &Tips.$tips.source.categories]}]})
+ (let [query (lib/query
+ (mt/metadata-provider)
+ (mt/mbql-query tips
+ {:joins [{:alias "Tips"
+ :source-table $$tips
+ :condition [:= $tips.source.categories &Tips.$tips.source.categories]}]}))
compiled (mongo.qp/mbql->native query)
let-lhs (-> compiled (get-in [:query 0 "$lookup" :let]) keys first)]
(is (and (not (str/includes? let-lhs "."))
@@ -451,16 +474,19 @@
:unit :week
:amount -1}}
86400000]}]}}
- (mongo.qp/compile-filter [:<
- [:+
- [:interval 1 :year]
- [:field "date-field"]
- 3600000
- [:interval -1 :month]]
- [:-
- [:absolute-datetime (t/local-date "2008-05-31")]
- [:interval -1 :week]
- 86400000]])))))))))
+ (mu/disable-enforcement
+ (compile-filter (lib/<
+ (lib/+
+ (lib/interval 1 :year)
+ (lib/ref {:lib/type :metadata/column
+ :base-type :type/Date
+ :name "date-field"})
+ 3600000 ; technically this is not valid MBQL
+ (lib/interval -1 :month))
+ (lib/-
+ (lib/absolute-datetime (t/local-date "2008-05-31") :default)
+ (lib/interval -1 :week)
+ 86400000))))))))))))
(deftest ^:synchronized temporal-arithmetic-mongo-4-test
(mt/test-driver :mongo
@@ -470,11 +496,13 @@
(is (thrown-with-msg?
clojure.lang.ExceptionInfo
#"Date arithmetic not supported in versions before 5"
- (mongo.qp/compile-filter [:<
- [:+
- [:interval 1 :year]
- [:field "date-field"]]
- [:absolute-datetime (t/local-date "2008-05-31")]]))))))))
+ (compile-filter (lib/<
+ (lib/+
+ (lib/interval 1 :year)
+ (lib/ref {:lib/type :metadata/column
+ :base-type :type/Date
+ :name "date-field"}))
+ (lib/absolute-datetime (t/local-date "2008-05-31") :default))))))))))
(deftest ^:parallel datetime-math-tests
(mt/test-driver :mongo
@@ -588,13 +616,18 @@
:condition [:= &People.people.id $orders.user_id]
:fields :all}]}})
compiled (qp.compile/compile query)
- indices (reduce (fn [acc lookup-stage]
+ indices (reduce (fn [seen lookup-stage]
(let [let-var-name (-> (get-in lookup-stage ["$lookup" :let]) keys first)
;; Following expression ensures index is an integer.
- index (parse-long (re-find #"\d+$" let-var-name))]
+ index (some->> let-var-name (re-find #"\d+$") parse-long)]
+ (testing (format "lookup stage =\n%s\n" (pr-str lookup-stage))
+ (is (some? index)
+ "Should have an index in the :let expression in $lookup stage"))
;; Following expression tests that index is unique.
- (is (not (contains? acc index)))
- (conj acc index)))
+ (is (not (contains? seen index))
+ "Index number should not have been seen yet")
+ (cond-> seen
+ (some? index) (conj index))))
#{}
(filter #(contains? % "$lookup") (:query compiled)))]
(is (= #{1 2 3 4} indices))))))
diff --git a/modules/drivers/mongo/test/metabase/driver/mongo_test.clj b/modules/drivers/mongo/test/metabase/driver/mongo_test.clj
index b403bac2852..a0b857affde 100644
--- a/modules/drivers/mongo/test/metabase/driver/mongo_test.clj
+++ b/modules/drivers/mongo/test/metabase/driver/mongo_test.clj
@@ -748,7 +748,7 @@
(qp/process-query
{:database (mt/id)
:type :native
- :native {:projections [:count]
+ :native {:projections ["count"]
:query [{"$project" {"price" "$price"}}
{"$match" {"price" {"$eq" 1}}}
{"$group" {"_id" nil, "count" {"$sum" 1}}}
diff --git a/modules/drivers/mongo/test/metabase/test/data/mongo.clj b/modules/drivers/mongo/test/metabase/test/data/mongo.clj
index 08ae545ec16..87f72a87c33 100644
--- a/modules/drivers/mongo/test/metabase/test/data/mongo.clj
+++ b/modules/drivers/mongo/test/metabase/test/data/mongo.clj
@@ -103,7 +103,7 @@
(defmethod tx/count-with-template-tag-query :mongo
[_driver table-name field-name _param-type]
- {:projections [:count]
+ {:projections ["count"]
:query (json/encode
[{:$match {(name field-name) (json/raw-json-generator (format "{{%s}}" (name field-name)))}}
{:$group {"_id" nil, "count" {:$sum 1}}}
@@ -113,7 +113,7 @@
(defmethod tx/count-with-field-filter-query :mongo
[_driver table-name field-name]
- {:projections [:count]
+ {:projections ["count"]
:query (json/encode
[{:$match (json/raw-json-generator (format "{{%s}}" (name field-name)))}
{:$group {"_id" nil, "count" {:$sum 1}}}
diff --git a/src/metabase/driver.clj b/src/metabase/driver.clj
index c303c2db995..1e47fa24570 100644
--- a/src/metabase/driver.clj
+++ b/src/metabase/driver.clj
@@ -1132,8 +1132,9 @@
(first messages))
(defmulti mbql->native
- "Transpile an MBQL query into the appropriate native query form. `query` will match the schema for an MBQL query in
- [[metabase.legacy-mbql.schema/Query]]; this function should return a native query that conforms to that schema.
+ "Transpile an MBQL query into the appropriate native query form. `query` will match the schema for an MBQL 5 query in
+ `:metabase.lib.schema/query`; this function should return a compiled query that matches the
+ `:metabase.query-processor.compile/compiled` schema.
If the underlying query language supports remarks or comments, the driver should
use [[metabase.query-processor.util/query->remark]] to generate an appropriate message and include that in an
@@ -1284,31 +1285,36 @@
(defmulti substitute-native-parameters-in-stage-method
"Implementation for [[substitute-native-parameters-in-stage]]; avoid calling this directly and
call [[substitute-native-parameters-in-stage]] instead. Only use this for `defmethod` method implementations."
- {:added "0.62.0" :arglists '([driver metadata-providerable native-query-stage])}
+ {:added "0.62.0" :arglists '([driver query stage-number])}
dispatch-on-initialized-driver
:hierarchy #'hierarchy)
-(mu/defn substitute-native-parameters-in-stage :- ::lib.schema/stage.native
- "For drivers that support `:native-parameters`. Substitute parameters in a normalized 'inner' native query.
+(mu/defn substitute-native-parameters-in-stage :- ::lib.schema/query
+ "For drivers that support `:native-parameters`. Substitute parameters in a normalized native query stage, and return
+ the updated query.
- {:lib/type :mbql.stage/native
- :native \"SELECT count(*) FROM table WHERE id = {{param}}\"
- :template-tags {:param {:name \"param\", :display-name \"Param\", :type :number}}
- :parameters [{:type :number
- :target [:variable [:template-tag \"param\"]]
- :value 2}]}
+ {:lib/type :mbql.query/mbql
+ :stages [{:lib/type :mbql.stage/native
+ :native \"SELECT count(*) FROM table WHERE id = {{param}}\"
+ :template-tags {:param {:name \"param\", :display-name \"Param\", :type :number}}
+ :parameters [{:type :number
+ :target [:variable [:template-tag \"param\"]]
+ :value 2}]}]}
->
- {:native \"SELECT count(*) FROM table WHERE id = 2\", ...}
+ {:lib/type :mbql.query/mbql
+ :stages [{:lib/type :mbql.stage/native
+ :native \"SELECT count(*) FROM table WHERE id = 2\"
+ ...}]}
Much of the implementation for this method is shared across drivers and lives in the
`metabase.query-processor.parameters.*` namespaces. See the `:sql` and `:mongo` drivers for sample implementations
of this method. Driver-agnostic end-to-end native parameter tests live in
[[metabase.query-processor.parameters-test]] and other namespaces."
{:added "0.62.0"}
- [driver :- :keyword
- metadata-providerable :- ::lib.schema.metadata/metadata-providerable
- native-stage :- ::lib.schema/stage.native]
- (substitute-native-parameters-in-stage-method driver metadata-providerable native-stage))
+ [driver :- :keyword
+ query :- ::lib.schema/query
+ stage-number :- :int]
+ (substitute-native-parameters-in-stage-method driver query stage-number))
(defmulti default-field-order
"Return how fields should be sorted by default for this database."
diff --git a/src/metabase/driver/sql.clj b/src/metabase/driver/sql.clj
index fa9a36b7f84..fc6cbfef01b 100644
--- a/src/metabase/driver/sql.clj
+++ b/src/metabase/driver/sql.clj
@@ -15,7 +15,6 @@
[metabase.driver.util :as driver.u]
[metabase.lib.core :as lib]
[metabase.lib.schema :as lib.schema]
- [metabase.lib.schema.metadata :as lib.schema.metadata]
[metabase.lib.util :as lib.util]
[metabase.query-processor.error-type :as qp.error-type]
[metabase.query-processor.parameters.values :as params.values]
@@ -82,19 +81,23 @@
[driver native-form]
(sql.u/format-sql-and-fix-params driver native-form))
-(mu/defmethod driver/substitute-native-parameters-in-stage-method :sql :- ::lib.schema/stage.native
- [_driver :- :keyword
- metadata-providerable :- ::lib.schema.metadata/metadata-providerable
- {native-query :native, :as native-stage} :- ::lib.schema/stage.native]
- (let [params-map (params.values/stage->params-map metadata-providerable native-stage)
- referenced-card-ids (params.values/referenced-card-ids params-map)
- parsed-query (lib/parse-parameters native-query)
- [native-query params] (sql.params.substitute/substitute metadata-providerable parsed-query params-map)]
- (cond-> (assoc native-stage
- :native native-query
- :params params)
- (seq referenced-card-ids)
- (update :query-permissions/referenced-card-ids set/union referenced-card-ids))))
+(mu/defmethod driver/substitute-native-parameters-in-stage-method :sql :- ::lib.schema/query
+ [_driver :- :keyword
+ query :- ::lib.schema/query
+ stage-number :- :int]
+ (lib/update-query-stage
+ query
+ stage-number
+ (fn [{native-query :native, :as native-stage}]
+ (let [params-map (params.values/stage->params-map query native-stage)
+ referenced-card-ids (params.values/referenced-card-ids params-map)
+ parsed-query (lib/parse-parameters native-query)
+ [native-query params] (sql.params.substitute/substitute query parsed-query params-map)]
+ (cond-> (assoc native-stage
+ :native native-query
+ :params params)
+ (seq referenced-card-ids)
+ (update :query-permissions/referenced-card-ids set/union referenced-card-ids))))))
(defmulti json-field-length
"Return a HoneySQL expression that calculates the number of characters in a JSON field for a given driver.
diff --git a/src/metabase/driver/sql/query_processor.clj b/src/metabase/driver/sql/query_processor.clj
index 8c1f51eb6dd..dabdab21957 100644
--- a/src/metabase/driver/sql/query_processor.clj
+++ b/src/metabase/driver/sql/query_processor.clj
@@ -40,6 +40,8 @@
FROM ( SELECT * FROM some_table ) __mb_source"
"__mb_source")
+;; TODO (Cam 2026-08-11) "Inner query" is MBQL 4 terminology, since we're using MBQL 5 now, rename this to `stage` and
+;; rename all the `inner-query` function args & local variables to `stage` as well
(def ^:dynamic *inner-query*
"The INNER query currently being processed, for situations where we need to refer back to it."
nil)
@@ -131,14 +133,6 @@
;;; | Interface (Multimethods) |
;;; +----------------------------------------------------------------------------------------------------------------+
-#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
-(defmulti honey-sql-version
- "DEPRECATED: Prior to between 0.46.0 and 0.49.0, drivers could use either Honey SQL 1 or Honey SQL 2. In 0.49.0+, all
- drivers must use Honey SQL 2."
- {:arglists '(^Long [driver]), :added "0.46.0", :deprecated "0.49.0"}
- driver/dispatch-on-initialized-driver
- :hierarchy #'driver/hierarchy)
-
(defn inline-num
"Wrap number `n` in `:inline` when targeting Honey SQL 2."
{:added "0.46.0"}
@@ -1630,10 +1624,16 @@
;;; -------------------------------------------------- aggregation ---------------------------------------------------
(defn- aggregation-name
- [inner-query ag-clause]
+ [_inner-query ag-clause]
(or (::add/desired-alias (lib/options ag-clause))
(:name (lib/options ag-clause))
- (lib/column-name inner-query ag-clause)))
+ ;; TODO (Cam 2026-08-11) this won't work because [[lib/column-name]] takes a top-level query, not
+ ;; an "inner-query" (stage), so I'm commenting it out for now. Fortunately things must still be working even
+ ;; without this fallback.
+ ;;
+ ;; Once we update this code to take query + stage-number we should use [[driver-api/mbql-5-aggregation-name]]
+ ;; directly since it has basically the same logic
+ #_(lib/column-name inner-query ag-clause)))
(defmethod apply-top-level-clause [:sql :aggregation]
[driver _top-level-clause honeysql-form {aggregations :aggregation, :as inner-query}]
@@ -1690,7 +1690,9 @@
[:lower field])
pattern])
-(def ^:private StringValueOrFieldOrExpression
+(def ^:private ^{:deprecated "0.64.0"} LegacyStringValueOrFieldOrExpression
+ "Deprecated: use MBQL 5 going forward."
+ #_{:clj-kondo/ignore [:deprecated-var]}
[:or
[:and driver-api/mbql.schema.value
[:fn {:error/message "string value"} #(string? (second %))]]
@@ -1737,7 +1739,7 @@
"Generate pattern to match against in like clause. Lowercasing for case insensitive matching also happens here."
[driver
pre
- [type _ :as arg] :- StringValueOrFieldOrExpression
+ [type _ :as arg] :- #_{:clj-kondo/ignore [:deprecated-var]} LegacyStringValueOrFieldOrExpression
post
{:keys [case-sensitive] :or {case-sensitive true} :as _options}]
(if (= :value type)
diff --git a/src/metabase/driver/sql_jdbc/actions.clj b/src/metabase/driver/sql_jdbc/actions.clj
index 307e788260f..5ef42b2e166 100644
--- a/src/metabase/driver/sql_jdbc/actions.clj
+++ b/src/metabase/driver/sql_jdbc/actions.clj
@@ -476,15 +476,16 @@
:before nil
:after row}))))
+;; TODO (Cam 2026-07-23) Update this stuff to use MBQL 5 instead of legacy MBQL
(mu/defn- model-create! :- (result-schema [:map [:created-row driver-api/schema.actions.args.row]])
- [action context inputs :- [:sequential driver-api/mbql.schema.Query]]
- (let [database (inputs->db inputs)
+ [action context legacy-queries :- #_{:clj-kondo/ignore [:deprecated-var]} [:sequential driver-api/mbql.schema.Query]]
+ (let [database (inputs->db legacy-queries)
;; TODO it would be nice to make this 1 statement per table, instead of N.
;; we can rely on the table lock instead of the nested row transactions.
[errors diffs] (run-bulk-transaction!
{:database database
:proc (partial row-create!* action database)
- :coll inputs})]
+ :coll legacy-queries})]
(if (seq errors)
;; For backwards compatibility
(throw (:error (first errors)))
diff --git a/src/metabase/driver_api/core.clj b/src/metabase/driver_api/core.clj
index cc609050dad..700dcb99214 100644
--- a/src/metabase/driver_api/core.clj
+++ b/src/metabase/driver_api/core.clj
@@ -47,6 +47,7 @@
[metabase.query-processor.timezone :as qp.timezone]
[metabase.query-processor.util :as qp.util]
[metabase.query-processor.util.add-alias-info :as add]
+ [metabase.query-processor.util.add-alias-info.helpers]
[metabase.query-processor.util.relative-datetime :as qp.relative-datetime]
[metabase.query-processor.util.transformations.nest-breakouts :as qp.util.transformations.nest-breakouts]
[metabase.query-processor.writeback :as qp.writeback]
@@ -73,6 +74,7 @@
actions/violate-permission-constraint
actions/violate-unique-constraint
add/add-alias-info
+ metabase.query-processor.util.add-alias-info.helpers/mbql-5-aggregation-name
annotate/aggregation-name
annotate/base-type-inferer
annotate/merged-column-info
@@ -277,31 +279,31 @@
"::lib.schema.validate/error"
::lib.schema.validate/error)
-(def mbql.schema.DateTimeValue
+(def ^{:deprecated "0.63.0"} mbql.schema.DateTimeValue
"::mbql.s/DateTimeValue"
::mbql.s/DateTimeValue)
-(def mbql.schema.Aggregation
+(def ^{:deprecated "0.63.0"} mbql.schema.Aggregation
"::mbql.s/Aggregation"
::mbql.s/Aggregation)
-(def mbql.schema.OrderBy
+(def ^{:deprecated "0.63.0"} mbql.schema.OrderBy
"::mbql.s/OrderBy"
::mbql.s/OrderBy)
-(def mbql.schema.Query
+(def ^{:deprecated "0.63.0"} mbql.schema.Query
"::mbql.s/Query"
::mbql.s/Query)
-(def mbql.schema.value
+(def ^{:deprecated "0.63.0"} mbql.schema.value
"mbql.s/value"
::mbql.s/value)
-(def mbql.schema.field
+(def ^{:deprecated "0.63.0"} mbql.schema.field
"mbql.s/field"
::mbql.s/field)
-(def mbql.schema.FieldOrExpressionDef
+(def ^{:deprecated "0.63.0"} mbql.schema.FieldOrExpressionDef
"::mbql.s/FieldOrExpressionDef"
::mbql.s/FieldOrExpressionDef)
@@ -341,10 +343,10 @@
"Schema for the output of [[compile]]: `:metabase.query-processor.compile/query-with-compiled-query`"
::qp.compile/query-with-compiled-query)
-(def MBQLQuery
+(def ^{:deprecated "0.63.0"} MBQLQuery
"Schema for a legacy MBQL inner query."
::mbql.s/MBQLInnerQuery)
-(def Join
+(def ^{:deprecated "0.63.0"} Join
"Schema for a legacy MBQL join."
::mbql.s/Join)
diff --git a/src/metabase/legacy_mbql/util.cljc b/src/metabase/legacy_mbql/util.cljc
index 19625fde216..262e553bc1e 100644
--- a/src/metabase/legacy_mbql/util.cljc
+++ b/src/metabase/legacy_mbql/util.cljc
@@ -72,7 +72,7 @@
Throws an Exception when it encounters a unresolved source query (i.e., the `:source-table \"card__id\"`
form), because it cannot return an accurate result for a query that has not yet been preprocessed.
- Prefer [[metabase.lib.core/source-table-id]] going forward."
+ Prefer [[metabase.lib.core/primary-source-table-id]] going forward."
{:arglists '([outer-query]), :deprecated "0.57.0"}
[{{source-table-id :source-table, source-query :source-query} :query, query-type :type, :as query} :- [:maybe :map]]
(cond
diff --git a/src/metabase/lib/convert.cljc b/src/metabase/lib/convert.cljc
index 21070962aea..3775d99daba 100644
--- a/src/metabase/lib/convert.cljc
+++ b/src/metabase/lib/convert.cljc
@@ -462,6 +462,7 @@
{:query inner-query
:type :query})))
+;; TODO (Cam 2026-07-17) Deprecate this in 64
(defmulti ->legacy-MBQL
"Coerce something to legacy MBQL (the version of MBQL understood by the query processor and Metabase Lib v1) if it's
not already legacy MBQL."
diff --git a/src/metabase/lib/core.cljc b/src/metabase/lib/core.cljc
index f0f329e7188..64211e3310d 100644
--- a/src/metabase/lib/core.cljc
+++ b/src/metabase/lib/core.cljc
@@ -339,6 +339,12 @@
new-name :- :string]
(lib.expression/with-expression-name an-expression-clause new-name))
+;; TODO (Cam 2026-07-13) Give these wrappers like the other functions here
+(shared.ns/import-fns
+ [lib.expression
+ resolve-expression
+ value])
+
;; ### Expression Functions
;; These functions are quite generic, so they are re-exported directly. Each of these functions takes a number of
;; arguments and returns an expression clause.
diff --git a/src/metabase/lib/expression.cljc b/src/metabase/lib/expression.cljc
index b525284ecd4..83319a83051 100644
--- a/src/metabase/lib/expression.cljc
+++ b/src/metabase/lib/expression.cljc
@@ -19,6 +19,7 @@
[metabase.lib.schema.expression :as lib.schema.expression]
[metabase.lib.schema.expression.conditional :as lib.schema.expression.conditional]
[metabase.lib.schema.expression.temporal :as lib.schema.expression.temporal]
+ [metabase.lib.schema.literal :as lib.schema.literal]
[metabase.lib.schema.metadata :as lib.schema.metadata]
[metabase.lib.schema.temporal-bucketing :as lib.schema.temporal-bucketing]
[metabase.lib.temporal-bucket :as lib.temporal-bucket]
@@ -421,12 +422,25 @@
unit))
(mu/defn value :- ::lib.schema.expression/expression
- "Creates a `:value` clause for the `literal`. Converts bigint literals to strings for serialization purposes."
- [literal :- [:or :string number? :boolean [:fn u.number/bigint?]]]
- (let [base-type (lib.schema.expression/type-of-resolved literal)]
- (lib.options/ensure-uuid [:value
- {:base-type base-type, :effective-type base-type}
- (cond-> literal (u.number/bigint? literal) str)])))
+ "Generate a new `:value` clause, used to wrap wrap value literals to allow type information to be attached to them.
+ Mostly used by the query processor. Converts bigint literals to strings for serialization purposes.
+
+ Note that `:effective-type` is required in `opts`."
+ ([literal :- [:or :string number? :boolean [:fn {:error/message "big integer?"} u.number/bigint?]]]
+ (value nil literal))
+ ([opts :- [:maybe [:merge
+ [:ref ::lib.schema.literal/value.options]
+ ;; `:lib/uuid` and `:effective-type` are optional, as they will be added automatically
+ [:map
+ [:lib/uuid {:optional true} ::lib.schema.common/uuid]
+ [:effective-type {:optional true} ::lib.schema.common/base-type]]]]
+ literal]
+ (let [base-type (or (:base-type opts) (lib.schema.expression/type-of-resolved literal))
+ effective-type (or (:effective-type opts) base-type)]
+ (-> [:value
+ (merge {:base-type base-type, :effective-type effective-type} opts)
+ (cond-> literal (u.number/bigint? literal) str)]
+ lib.options/ensure-uuid))))
(mu/defn expression-metadata :- ::lib.schema.metadata/column
"Return column metadata for an `expression-definition` MBQL clause."
diff --git a/src/metabase/lib/js.cljs b/src/metabase/lib/js.cljs
index 5e6a22174bb..4e1758d18b5 100644
--- a/src/metabase/lib/js.cljs
+++ b/src/metabase/lib/js.cljs
@@ -85,7 +85,6 @@
[metabase.lib.native :as lib.native]
[metabase.lib.normalize :as lib.normalize]
[metabase.lib.order-by :as lib.order-by]
- [metabase.lib.query :as lib.query]
[metabase.lib.query.test-spec :as lib.query.test-spec]
[metabase.lib.schema.ref :as lib.schema.ref]
[metabase.lib.types.isa :as lib.types.isa]
@@ -198,7 +197,7 @@
> **Code health:** Legacy. This has many legitimate uses (as of March 2024), but we should aim to reduce the places
where a legacy query is still needed. Consider if it's practical to port the consumer of this legacy query to MBQL 5."
[query-map]
- (-> (lib.query/->legacy-MBQL query-map)
+ (-> (lib.convert/->legacy-MBQL query-map)
fix-namespaced-values (clj->js :keyword-fn u/qualified-name)))
(defn ^:export append-stage
diff --git a/src/metabase/lib/query.cljc b/src/metabase/lib/query.cljc
index e6e92ec028f..0679f825b2b 100644
--- a/src/metabase/lib/query.cljc
+++ b/src/metabase/lib/query.cljc
@@ -321,11 +321,6 @@
lib.convert/->mbql5
(query metadata-providerable)))
-(defn ->legacy-MBQL
- "Convert the MBQL 5 `a-query` into a legacy MBQL query."
- [a-query]
- (-> a-query lib.convert/->legacy-MBQL))
-
(mu/defn with-different-table :- ::lib.schema/query
"Changes an existing query to use a different source table or card.
Can be passed an integer table id or a legacy `card__<id>` string."
diff --git a/src/metabase/lib/schema/expression/conditional.cljc b/src/metabase/lib/schema/expression/conditional.cljc
index 43b399c5e03..5468fefc985 100644
--- a/src/metabase/lib/schema/expression/conditional.cljc
+++ b/src/metabase/lib/schema/expression/conditional.cljc
@@ -76,6 +76,9 @@
;;; believe it or not, a `:case` clause really has the syntax [:case {} [[pred1 expr1] [pred2 expr2] ...]]
;;; `:if` is an alias to `:case`
+;;
+;; In MBQL 4, the optional default value is part of the options map; in MBQL 5 it is an optional final standalone
+;; value.
(doseq [tag [:case :if]]
(mbql-clause/define-mbql-clause
tag
diff --git a/src/metabase/lib/schema/expression/temporal.cljc b/src/metabase/lib/schema/expression/temporal.cljc
index 1187f35f849..833e3b77b6c 100644
--- a/src/metabase/lib/schema/expression/temporal.cljc
+++ b/src/metabase/lib/schema/expression/temporal.cljc
@@ -290,12 +290,12 @@
[:unit [:? [:schema [:ref ::relative-datetime.unit]]]])
(mbql-clause/define-tuple-mbql-clause :time :- :type/Time
- #_:timestr [:schema [:ref ::expression/string]]
+ #_:timestr [:schema [:ref ::literal/time]]
#_:unit [:or
[:= {:decode/normalize common/normalize-keyword-lower} :default]
[:ref ::temporal-bucketing/unit.time.interval]])
-;;; this has some stuff that's missing from [[::temporal-bucketing/unit.date-time.extract]], like `:week-of-year-iso`
+;;; TODO this has some stuff that's missing from [[::temporal-bucketing/unit.date-time.extract]], like `:week-of-year-iso`
(mr/def ::temporal-extract.unit
[:enum
{:decode/normalize common/normalize-keyword-lower}
diff --git a/src/metabase/lib/schema/mbql_clause.cljc b/src/metabase/lib/schema/mbql_clause.cljc
index f3b5f2055a0..dce3984c064 100644
--- a/src/metabase/lib/schema/mbql_clause.cljc
+++ b/src/metabase/lib/schema/mbql_clause.cljc
@@ -18,7 +18,9 @@
"Given an MBQL clause tag like `:starts-with`, return the name of the schema we'll register for it, e.g.
`:mbql.clause/starts-with`."
[tag]
- (keyword "mbql.clause" (name tag)))
+ (if (qualified-keyword? tag)
+ tag
+ (keyword "mbql.clause" (name tag))))
(defn registered-tags
"Snapshot of every MBQL clause tag currently registered via [[define-mbql-clause]]
@@ -39,7 +41,7 @@
(defn- clause-schema
"Build the schema for `::clause`, a `:multi` schema that maps MBQL clause tag -> the schema
- in [[clause-schema-registry]]."
+ in global schema registry in [[metabase.util.malli.registry/registry]]."
[]
(into [:multi
{:dispatch common/mbql-clause-tag
@@ -78,7 +80,7 @@
[:= :is-null]
::common/options
[:ref :metabase.lib.schema.expression/expression]])"
- ([tag :- simple-keyword?
+ ([tag :- keyword?
schema]
(let [schema-name (tag->registered-schema-name tag)]
(mr/def schema-name schema)
@@ -89,7 +91,7 @@
(swap! tag-registry conj tag)))
nil)
- ([tag :- simple-keyword?
+ ([tag :- keyword?
_arrow :- [:= :-]
return-type :- ::expression/base-type
schema]
diff --git a/src/metabase/lib/value.cljc b/src/metabase/lib/value.cljc
new file mode 100644
index 00000000000..0e5a11b4624
--- /dev/null
+++ b/src/metabase/lib/value.cljc
@@ -0,0 +1,18 @@
+(ns metabase.lib.value
+ (:require
+ [metabase.lib.schema.common :as lib.schema.common]
+ [metabase.lib.schema.literal :as lib.schema.literal]
+ [metabase.util.malli :as mu]))
+
+(mu/defn value :- :mbql.clause/value
+ "Generate a new `:value` clause, used to wrap wrap value literals to allow type information to be attached to them.
+ Mostly used by the query processor.
+
+ Note that `:effective-type` is required in `opts`."
+ [opts :- [:maybe [:merge
+ [:ref ::lib.schema.literal/value.options]
+ ;; `:lib/uuid` is optional, as it will be added automatically
+ [:map
+ [:lib/uuid {:optional true} ::lib.schema.common/uuid]]]]
+ v]
+ [:value (assoc opts :lib/uuid (str (random-uuid))) v])
diff --git a/src/metabase/query_processor/middleware/annotate.clj b/src/metabase/query_processor/middleware/annotate.clj
index 4d94cda0cb2..70859829ee4 100644
--- a/src/metabase/query_processor/middleware/annotate.clj
+++ b/src/metabase/query_processor/middleware/annotate.clj
@@ -157,6 +157,7 @@
;;; These are only for convenience for drivers that used to use stuff in annotate directly -- we can remove it once we
;;; convert drivers to MBQL 5.
+#_{:clj-kondo/ignore [:deprecated-var]}
(p/import-vars
[metabase.query-processor.middleware.annotate.legacy-helper-fns
aggregation-name
diff --git a/src/metabase/query_processor/middleware/annotate/legacy_helper_fns.clj b/src/metabase/query_processor/middleware/annotate/legacy_helper_fns.clj
index aa309c0ff0c..94db46258fe 100644
--- a/src/metabase/query_processor/middleware/annotate/legacy_helper_fns.clj
+++ b/src/metabase/query_processor/middleware/annotate/legacy_helper_fns.clj
@@ -43,7 +43,10 @@
"Return an appropriate aggregation name/alias *used inside a query* for an `:aggregation` subclause (an aggregation
or expression). Takes an options map as schema won't support passing keypairs directly as a varargs.
- These names are also used directly in queries, e.g. in the equivalent of a SQL `AS` clause."
+ These names are also used directly in queries, e.g. in the equivalent of a SQL `AS` clause.
+
+ DEPRECATED: use [[mbql-5-aggregation-name]] going forward."
+ {:deprecated "0.64.0"}
[legacy-inner-query :- :map
legacy-ag-clause]
(let [ag-clause (lib/->mbql5 legacy-ag-clause)]
diff --git a/src/metabase/query_processor/middleware/fetch_source_query.clj b/src/metabase/query_processor/middleware/fetch_source_query.clj
index a8ddfcdd68c..175f47d7548 100644
--- a/src/metabase/query_processor/middleware/fetch_source_query.clj
+++ b/src/metabase/query_processor/middleware/fetch_source_query.clj
@@ -25,22 +25,23 @@
;;; TODO -- consider whether [[normalize-card-query]] should be moved into [[metabase.lib.card]], seems like it would
;;; make sense but it would involve teasing out some QP-specific stuff to make it work.
-(defn- fix-mongodb-first-stage
+(mu/defn- fix-mongodb-first-stage :- ::lib.schema/stages
"MongoDB native queries consist of a collection and a pipelne (query).
TODO -- it's not great that this code lives here. This should be part of the MongoDB driver. We should NOT be
hardcoding driver-specific behavior in generic QP middleware."
- [[first-stage & more]]
- (let [first-stage (cond-> first-stage
- (and (= driver/*driver* :mongo)
- (= (:lib/type first-stage) :mbql.stage/native))
- (update :native (fn [x]
- (if (map? x)
- x
- {:collection (:collection first-stage)
- :projections (:projections first-stage)
- :query x}))))]
- (cons first-stage more)))
+ [stages :- ::lib.schema/stages]
+ (letfn [(update-first-stage [first-stage]
+ (cond-> first-stage
+ (and (= driver/*driver* :mongo)
+ (= (:lib/type first-stage) :mbql.stage/native))
+ (update :native (fn [x]
+ (if (map? x)
+ x
+ {:collection (:collection first-stage)
+ :projections (:projections first-stage)
+ :query x})))))]
+ (update (vec stages) 0 update-first-stage)))
(def ^:private qp-owned-stage-keys
[:persisted-info/native
diff --git a/src/metabase/query_processor/middleware/parameters.clj b/src/metabase/query_processor/middleware/parameters.clj
index 7b149ece04f..e479029376d 100644
--- a/src/metabase/query_processor/middleware/parameters.clj
+++ b/src/metabase/query_processor/middleware/parameters.clj
@@ -27,8 +27,11 @@
stage
(let [f (case (:lib/type stage)
:mbql.stage/mbql qp.mbql/expand
- :mbql.stage/native (fn [query _path stage]
- (qp.native/expand-stage query stage)))
+ :mbql.stage/native (mu/fn :- ::lib.schema/stage.native
+ [query path _stage]
+ (let [{query* :query, :keys [stage-number]} (lib.walk/query-for-path query path)]
+ (-> (qp.native/expand-stage query* stage-number)
+ (lib/query-stage stage-number)))))
expanded (f query path stage)]
(dissoc expanded :parameters :template-tags))))
diff --git a/src/metabase/query_processor/middleware/parameters/native.clj b/src/metabase/query_processor/middleware/parameters/native.clj
index 27302b18027..196b7d4f2de 100644
--- a/src/metabase/query_processor/middleware/parameters/native.clj
+++ b/src/metabase/query_processor/middleware/parameters/native.clj
@@ -27,19 +27,17 @@
(:require
[metabase.driver :as driver]
[metabase.driver.util :as driver.u]
+ [metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.lib.schema :as lib.schema]
- [metabase.lib.schema.metadata :as lib.schema.metadata]
[metabase.util.malli :as mu]))
-(mu/defn expand-stage :- ::lib.schema/stage.native
+(mu/defn expand-stage :- ::lib.schema/query
"Expand parameters inside an *inner* native `query`. Not recursive -- recursive transformations are handled in
the `middleware.parameters` functions that invoke this function."
- [metadata-providerable :- ::lib.schema.metadata/metadata-providerable
- stage :- ::lib.schema/stage.native]
- (if-not (driver.u/supports? driver/*driver* :native-parameters (lib.metadata/database metadata-providerable))
- stage
- (let [substituted-stage (driver/substitute-native-parameters-in-stage driver/*driver* metadata-providerable stage)]
- (->
- substituted-stage
- (dissoc :parameters :template-tags)))))
+ [query :- ::lib.schema/query
+ stage-number :- :int]
+ (if-not (driver.u/supports? driver/*driver* :native-parameters (lib.metadata/database query))
+ query
+ (-> (driver/substitute-native-parameters-in-stage driver/*driver* query stage-number)
+ (lib/update-query-stage stage-number dissoc :parameters :template-tags))))
diff --git a/src/metabase/query_processor/middleware/wrap_value_literals.clj b/src/metabase/query_processor/middleware/wrap_value_literals.clj
index 6d7f4ac96c6..e281c2ce350 100644
--- a/src/metabase/query_processor/middleware/wrap_value_literals.clj
+++ b/src/metabase/query_processor/middleware/wrap_value_literals.clj
@@ -23,10 +23,6 @@
(set! *warn-on-reflection* true)
-(mu/defn- value :- :mbql.clause/value
- [info :- :map v]
- [:value (assoc info :lib/uuid (str (random-uuid))) v])
-
(defn- type-info-from-col [col]
(when col
(merge
@@ -60,11 +56,11 @@
(defmethod add-type-info nil
[_ info & _]
- (value info nil))
+ (lib/value info nil))
(defmethod add-type-info Object
[this info & _]
- (value info this))
+ (lib/value info this))
(derive LocalDate ::->absolute-datetime)
(derive LocalDateTime ::->absolute-datetime)
@@ -214,7 +210,7 @@
(seq s))
(let [effective-type ((some-fn :effective-type :base-type) col)]
(parse-temporal-string-literal effective-type s (or unit :default)))
- (value col s)))
+ (lib/value col s)))
;;; -------------------------------------------- wrap-literals-in-clause ---------------------------------------------
diff --git a/src/metabase/query_processor/store.clj b/src/metabase/query_processor/store.clj
index a1f2b7483e8..b760800f3ae 100644
--- a/src/metabase/query_processor/store.clj
+++ b/src/metabase/query_processor/store.clj
@@ -121,8 +121,9 @@
(let [ks (into [(list 'quote (gensym (str (name (ns-name *ns*)) "/misc-cache-")))] (u/one-or-many k-or-ks))]
`(cached-fn ~ks (fn [] ~@body))))
+;; TODO (Cam 2026-07-17) Deprecate this in 64
(mu/defn metadata-provider :- ::lib.schema.metadata/metadata-provider
- "Get the [[metabase.lib.metadata.protocols/MetadataProvider]] that should be used inside the QP. "
+ "Get the [[metabase.lib.metadata.protocols/MetadataProvider]] that should be used inside the QP."
[]
(or (miscellaneous-value [::metadata-provider])
(throw (ex-info "QP Store Metadata Provider is not initialized yet; initialize it with `qp.store/with-metadata-provider`."
diff --git a/src/metabase/query_processor/util/add_alias_info/helpers.clj b/src/metabase/query_processor/util/add_alias_info/helpers.clj
new file mode 100644
index 00000000000..18f042c9c71
--- /dev/null
+++ b/src/metabase/query_processor/util/add_alias_info/helpers.clj
@@ -0,0 +1,21 @@
+(ns metabase.query-processor.util.add-alias-info.helpers
+ (:require
+ [metabase.lib.core :as lib]
+ [metabase.lib.schema :as lib.schema]
+ [metabase.lib.schema.common :as lib.schema.common]
+ [metabase.lib.schema.mbql-clause :as lib.schema.mbql-clause]
+ [metabase.query-processor.util.add-alias-info :as-alias add]
+ [metabase.util.malli :as mu]))
+
+(mu/defn mbql-5-aggregation-name :- ::lib.schema.common/non-blank-string
+ "Return an appropriate aggregation name/alias *used inside a query* for an `:aggregation` subclause (an aggregation
+ or expression). Takes an options map as schema won't support passing keypairs directly as a varargs.
+
+ These names are also used directly in queries, e.g. in the equivalent of a SQL `AS` clause."
+ [query :- ::lib.schema/query
+ stage-number :- :int
+ ag-clause :- ::lib.schema.mbql-clause/clause]
+ (let [opts (lib/options ag-clause)]
+ (or (::add/desired-alias opts)
+ (:name opts)
+ (lib/column-name query stage-number ag-clause))))
diff --git a/src/metabase/query_processor/util/transformations/nest_breakouts.clj b/src/metabase/query_processor/util/transformations/nest_breakouts.clj
index ae16f0731dd..854d010845a 100644
--- a/src/metabase/query_processor/util/transformations/nest_breakouts.clj
+++ b/src/metabase/query_processor/util/transformations/nest_breakouts.clj
@@ -8,6 +8,7 @@
[metabase.lib.core :as lib]
[metabase.lib.equality :as lib.equality]
[metabase.lib.schema :as lib.schema]
+ [metabase.lib.schema.expression :as lib.schema.expression]
[metabase.lib.schema.metadata :as lib.schema.metadata]
[metabase.lib.schema.util :as lib.schema.util]
[metabase.lib.walk :as lib.walk]
@@ -111,9 +112,16 @@
temporal-unit)
granularity))))
-(defn finest-temporal-breakout-index
- "Returns the index of leftmost breakout among the breakouts with the finest temporal granularity."
- [breakouts option-index]
+(mu/defn finest-temporal-breakout-index
+ "Returns the index of leftmost breakout among the breakouts with the finest temporal granularity.
+
+ `option-index` = index within an MBQL clause to look for an options map. Use `2` for Legacy MBQL (MBQL 4) and `1`
+ for MBQL 5.
+
+ TODO (Cam 2026-07-24) remove `option-index` as an argument, I believe it was only here for compatibility with either
+ MBQL 4 or 5, not that we're using MBQL 5 everywhere we can drop compatibility for 4."
+ [breakouts :- [:maybe [:sequential ::lib.schema.expression/expression]]
+ option-index :- pos-int?]
(loop [bs (seq breakouts)
i 0
min-granularity (inc (apply max (vals granularity)))
diff --git a/test/metabase/driver/impl_test.clj b/test/metabase/driver/impl_test.clj
index be6241babc7..2f6bce4cfda 100644
--- a/test/metabase/driver/impl_test.clj
+++ b/test/metabase/driver/impl_test.clj
@@ -101,7 +101,7 @@
[0 42 0])
0)))
-(deftest driver-multimethods-in-changelog-test
+(deftest ^:parallel driver-multimethods-in-changelog-test
(let [metadatas (collect-metadatas)
version->section-text (parse-drivers-changelog)]
(doseq [m metadatas]
diff --git a/test/metabase/driver/sql/parameters/substitute_test.clj b/test/metabase/driver/sql/parameters/substitute_test.clj
index d867c22b48c..ac576d31b3a 100644
--- a/test/metabase/driver/sql/parameters/substitute_test.clj
+++ b/test/metabase/driver/sql/parameters/substitute_test.clj
@@ -534,10 +534,8 @@
(merge {:database (meta/id)
:type :native}
query)))
- (lib/update-query-stage 0 (fn [stage]
- (-> stage
- (update :parameters concat parameters)
- (->> (qp.native/expand-stage mp)))))
+ (lib/update-query-stage 0 update :parameters concat parameters)
+ (qp.native/expand-stage 0)
lib/->legacy-MBQL))))
(defn- expand*
diff --git a/test/metabase/lib/breakout_test.cljc b/test/metabase/lib/breakout_test.cljc
index 1a8117c5a3c..3f871d9d6dc 100644
--- a/test/metabase/lib/breakout_test.cljc
+++ b/test/metabase/lib/breakout_test.cljc
@@ -5,7 +5,6 @@
[medley.core :as m]
[metabase.lib.breakout :as lib.breakout]
[metabase.lib.core :as lib]
- [metabase.lib.query :as lib.query]
[metabase.lib.test-metadata :as meta]
[metabase.lib.test-util :as lib.tu]
[metabase.lib.test-util.mocks-31368 :as lib.tu.mocks-31368]
@@ -229,7 +228,7 @@
(let [base-query (lib/query meta/metadata-provider (meta/table-metadata :people))
column (meta/field-metadata :people :birth-date)
query (lib/breakout (->> (lib/breakout base-query (lib/with-temporal-bucket column :year))
- (lib.query/->legacy-MBQL)
+ lib/->legacy-MBQL
(lib/query meta/metadata-provider))
(lib/with-temporal-bucket column :year))
breakouts (lib/breakouts query)]
@@ -238,7 +237,7 @@
(let [base-query (lib/query meta/metadata-provider (meta/table-metadata :people))
column (meta/field-metadata :people :birth-date)
query (lib/breakout (->> (lib/breakout base-query (lib/with-temporal-bucket column :year))
- (lib.query/->legacy-MBQL)
+ lib/->legacy-MBQL
(lib/query meta/metadata-provider))
(lib/with-temporal-bucket column :month))
breakouts (lib/breakouts query)]
diff --git a/test/metabase/lib/remove_replace_test.cljc b/test/metabase/lib/remove_replace_test.cljc
index 907e07a41a4..e0a009ef784 100644
--- a/test/metabase/lib/remove_replace_test.cljc
+++ b/test/metabase/lib/remove_replace_test.cljc
@@ -10,7 +10,6 @@
[metabase.lib.join :as lib.join]
[metabase.lib.metadata :as lib.metadata]
[metabase.lib.options :as lib.options]
- [metabase.lib.query :as lib.query]
[metabase.lib.remove-replace :as lib.remove-replace]
[metabase.lib.schema.expression :as lib.schema.expression]
[metabase.lib.test-metadata :as meta]
@@ -465,7 +464,7 @@
(lib/breakout (lib/with-temporal-bucket column :year))
(lib/breakout (lib/with-temporal-bucket column :month)))
query (->> query
- (lib.query/->legacy-MBQL)
+ lib/->legacy-MBQL
(lib/query meta/metadata-provider))]
(is (= query
(lib/replace-clause query
diff --git a/test/metabase/query_processor/explicit_joins_test.clj b/test/metabase/query_processor/explicit_joins_test.clj
index 7163e09da8f..77c8cd28d0e 100644
--- a/test/metabase/query_processor/explicit_joins_test.clj
+++ b/test/metabase/query_processor/explicit_joins_test.clj
@@ -1722,7 +1722,7 @@
(deftest ^:parallel dangling-join-condition-lhs-errors-if-fuzzy-matched-to-rhs-test
(testing (str "When upstream changes leave a dangling ref in a join condition LHS, QP throws if it is fuzzy-matched"
- "to a column from the RHS of the same join (#67667)")
+ " to a column from the RHS of the same join (#67667)")
(let [mp meta/metadata-provider
;; Q1(a) is a plain query of Products
q1a (lib/query mp (lib.metadata/table mp (meta/id :products)))
diff --git a/test/metabase/query_processor/filter_test.clj b/test/metabase/query_processor/filter_test.clj
index 90bae8d0f99..a9b719e1ee5 100644
--- a/test/metabase/query_processor/filter_test.clj
+++ b/test/metabase/query_processor/filter_test.clj
@@ -1037,25 +1037,26 @@
:order-by [[$id :asc]]
:limit 1}))))))))
-(deftest ^:parallel order-by-nulls-test
+(deftest order-by-nulls-test
(testing "Check that we can sort by numeric columns that contain NULLs (#6615)"
(mt/dataset daily-bird-counts
(mt/test-drivers (mt/normal-drivers)
- ;; the rows returned should be the ones with a nil count, in increasing ID order
- (is (= (if (mt/sorts-nil-first? driver/*driver* :type/Integer)
- ;; if nils come first, we expect the first three rows having a nil count, in id ascending order
- [[1 "2018-09-20T00:00:00Z" nil]
- [8 "2018-09-27T00:00:00Z" nil]
- [15 "2018-10-04T00:00:00Z" nil]]
- ;; if nils come last, we expect the first three rows having a count of 0, in id ascending order
- [[2 "2018-09-21T00:00:00Z" 0]
- [3 "2018-09-22T00:00:00Z" 0]
- [9 "2018-09-28T00:00:00Z" 0]])
- (mt/formatted-rows
- [int identity int]
- (mt/run-mbql-query bird-count
- {:order-by [[:asc $count] [:asc $id]]
- :limit 3}))))))))
+ (mt/with-temporary-setting-values [report-timezone "UTC"]
+ ;; the rows returned should be the ones with a nil count, in increasing ID order
+ (is (= (if (mt/sorts-nil-first? driver/*driver* :type/Integer)
+ ;; if nils come first, we expect the first three rows having a nil count, in id ascending order
+ [[1 "2018-09-20T00:00:00Z" nil]
+ [8 "2018-09-27T00:00:00Z" nil]
+ [15 "2018-10-04T00:00:00Z" nil]]
+ ;; if nils come last, we expect the first three rows having a count of 0, in id ascending order
+ [[2 "2018-09-21T00:00:00Z" 0]
+ [3 "2018-09-22T00:00:00Z" 0]
+ [9 "2018-09-28T00:00:00Z" 0]])
+ (mt/formatted-rows
+ [int identity int]
+ (mt/run-mbql-query bird-count
+ {:order-by [[:asc $count] [:asc $id]]
+ :limit 3})))))))))
(deftest filter-on-specific-date-test
(testing (str "Filtering on a specific date (DATE column) should work correctly regardless of report timezone/DB"
diff --git a/test/metabase/query_processor/middleware/annotate/legacy_helper_fns_test.clj b/test/metabase/query_processor/middleware/annotate/legacy_helper_fns_test.clj
index abaf2e2122b..0f6ef355d50 100644
--- a/test/metabase/query_processor/middleware/annotate/legacy_helper_fns_test.clj
+++ b/test/metabase/query_processor/middleware/annotate/legacy_helper_fns_test.clj
@@ -17,4 +17,5 @@
:aggregation [[:count]]}]
(qp.store/with-metadata-provider meta/metadata-provider
(is (= "count"
+ #_{:clj-kondo/ignore [:deprecated-var]}
(annotate.legacy-helper-fns/aggregation-name inner-query [:count]))))))
diff --git a/test/metabase/query_processor/middleware/parameters/native_test.clj b/test/metabase/query_processor/middleware/parameters/native_test.clj
index 2dcddd81ddc..202e1a7d1a7 100644
--- a/test/metabase/query_processor/middleware/parameters/native_test.clj
+++ b/test/metabase/query_processor/middleware/parameters/native_test.clj
@@ -36,7 +36,8 @@
(is (malli= [:map
[:native ms/NonBlankString]
[:params [:= ["G%"]]]]
- (qp.native/expand-stage mp (lib/query-stage query 0))))))))
+ (-> (qp.native/expand-stage query 0)
+ (lib/query-stage 0))))))))
(deftest ^:parallel native-query-with-card-template-tag-include-referenced-card-ids-test
(mt/test-drivers (mt/normal-drivers-with-feature :native-parameters :nested-queries :native-parameter-card-reference)
@@ -66,4 +67,5 @@
;; this SHOULD NOT include `1`, because Card 1 is only referenced indirectly; if you have permissions to run
;; Card 2 that should be sufficient to run it even if it references Card 1 (see #15131)
(is (=? {:query-permissions/referenced-card-ids #{Integer/MAX_VALUE 2}}
- (qp.native/expand-stage mp (lib/query-stage query 0))))))))
+ (-> (qp.native/expand-stage query 0)
+ (lib/query-stage 0))))))))
diff --git a/test/metabase/test/data.clj b/test/metabase/test/data.clj
index 8dbed04691c..bc70661e2d3 100644
--- a/test/metabase/test/data.clj
+++ b/test/metabase/test/data.clj
@@ -274,7 +274,8 @@
(data/dataset (get-dataset-definition) ...)"
{:style/indent :defn}
[dataset & body]
- `(t/testing (colorize/magenta ~(str (if (symbol? dataset)
+ `(t/testing (colorize/magenta ~(str \newline
+ (if (symbol? dataset)
(format "using %s dataset" dataset)
"using inline dataset")
\newline))