Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

nest-query => Lib (#64940)

Cam Saul15b31fb8342f
diff --git a/docs/developers-guide/driver-changelog.md b/docs/developers-guide/driver-changelog.md
index 89fcdf8e2f8..645947c27ef 100644
--- a/docs/developers-guide/driver-changelog.md
+++ b/docs/developers-guide/driver-changelog.md
@@ -48,6 +48,12 @@ title: Driver interface changelog
you can pass to [[metabase.lib.core/order-by]] -- for example a Lib `:metadata/column` or an [MBQL 5]
`metabase.driver-api.core/order-by-clause`.
+- `metabase.driver-api.core/nest-query` no longer automatically calls `metabase.driver-api.core/add-alias-info` on its
+ results, but it also no longer expect its input to have this information. If you were using both of these tools in
+ your driver, make change the order in which the are applied so `nest-query` happens first, followed by
+ `add-alias-info`. Note that drivers deriving from `:sql` do not need to make any changes, since this is done by the
+ base `:sql` driver implementation.
+
## Metabase 0.56.3
- Added the driver multi-method `driver/describe-database*` that drivers should now implement instead of `driver/describe-database`.
diff --git a/modules/drivers/sqlite/test/metabase/driver/sqlite_test.clj b/modules/drivers/sqlite/test/metabase/driver/sqlite_test.clj
index 64d018ca526..8a8d524a959 100644
--- a/modules/drivers/sqlite/test/metabase/driver/sqlite_test.clj
+++ b/modules/drivers/sqlite/test/metabase/driver/sqlite_test.clj
@@ -214,14 +214,13 @@
(deftest ^:parallel duplicate-identifiers-test
(testing "Make sure duplicate identifiers (even with different cases) get unique aliases"
(mt/test-driver :sqlite
- (is (= '{:select [source.CATEGORY_2 AS CATEGORY_2
+ (is (= '{:select [source.CATEGORY AS CATEGORY
COUNT (*) AS count]
- :from [{:select [products.category AS category
- products.category || ? AS CATEGORY_2]
+ :from [{:select [products.category || ? AS CATEGORY]
:from [products]}
AS source]
- :group-by [source.CATEGORY_2]
- :order-by [source.CATEGORY_2 ASC]
+ :group-by [source.CATEGORY]
+ :order-by [source.CATEGORY ASC]
:limit [1]}
(sql.qp-test-util/query->sql-map
(mt/mbql-query products
diff --git a/src/metabase/driver/sql/query_processor.clj b/src/metabase/driver/sql/query_processor.clj
index 322d5b8d900..946e23658c0 100644
--- a/src/metabase/driver/sql/query_processor.clj
+++ b/src/metabase/driver/sql/query_processor.clj
@@ -2045,8 +2045,8 @@
[_driver inner-query]
(-> inner-query
maybe-nest-breakouts-in-queries-with-window-fn-aggregations
- driver-api/add-alias-info
- driver-api/nest-expressions))
+ driver-api/nest-expressions
+ driver-api/add-alias-info))
(mu/defn mbql->honeysql :- [:or :map [:tuple [:= :inline] :map]]
"Build the HoneySQL form we will compile to SQL and execute."
diff --git a/src/metabase/driver_api/core.clj b/src/metabase/driver_api/core.clj
index da29b92cb50..20937bfc5c8 100644
--- a/src/metabase/driver_api/core.clj
+++ b/src/metabase/driver_api/core.clj
@@ -48,7 +48,6 @@
[metabase.query-processor.timezone :as qp.timezone]
[metabase.query-processor.util :as qp.util]
[metabase.query-processor.util.add-alias-info :as add]
- [metabase.query-processor.util.nest-query :as nest-query]
[metabase.query-processor.util.relative-datetime :as qp.relative-datetime]
[metabase.query-processor.util.transformations.nest-breakouts :as qp.util.transformations.nest-breakouts]
[metabase.query-processor.writeback :as qp.writeback]
@@ -140,8 +139,8 @@
mdb/spec
metabase.driver-api.impl/cached
metabase.driver-api.impl/dispatch-by-clause-name-or-class
+ metabase.driver-api.impl/nest-expressions
mi/instance-of?
- nest-query/nest-expressions
premium-features/is-hosted?
qp.compile/compile
qp.debug/debug>
diff --git a/src/metabase/driver_api/impl.clj b/src/metabase/driver_api/impl.clj
index 3436d25c384..e3b2b92bf7f 100644
--- a/src/metabase/driver_api/impl.clj
+++ b/src/metabase/driver_api/impl.clj
@@ -1,11 +1,15 @@
(ns metabase.driver-api.impl
- {:clj-kondo/config '{:linters
- ;; this is actually ok here since this is a drivers namespace
- {:discouraged-namespace {metabase.query-processor.store {:level :off}}}}}
+ {:clj-kondo/config
+ '{:linters
+ {:discouraged-namespace {metabase.legacy-mbql.util {:level :off}
+ metabase.query-processor.store {:level :off}}
+ :deprecated-namespace {:exclude [metabase.query-processor.store]}}}}
(:require
- ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.util :as mbql.u]
+ [metabase.legacy-mbql.util :as mbql.u]
+ [metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
- ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store]
+ [metabase.query-processor.store :as qp.store]
+ [metabase.query-processor.util.nest-query :as nest-query]
[metabase.util :as u]
[toucan2.core :as t2]))
@@ -53,3 +57,17 @@
(type x))))
([x _]
(dispatch-by-clause-name-or-class x)))
+
+(defn nest-expressions
+ "Wrapper around [[metabase.query-processor.util.nest-query/nest-expressions]] to allow it to work with legacy MBQL
+ inner queries. This is a temporary placeholder until we start migrating drivers to use Lib/MBQL 5 directly."
+ [lib-query-or-legacy-inner-query]
+ (if (:lib/type lib-query-or-legacy-inner-query)
+ (nest-query/nest-expressions lib-query-or-legacy-inner-query)
+ #_{:clj-kondo/ignore [:deprecated-var :discouraged-var]}
+ (-> lib-query-or-legacy-inner-query
+ (->> (lib/query-from-legacy-inner-query (qp.store/metadata-provider)
+ (:id (lib.metadata/database (qp.store/metadata-provider)))))
+ nest-query/nest-expressions
+ lib/->legacy-MBQL
+ :query)))
diff --git a/src/metabase/lib/equality.cljc b/src/metabase/lib/equality.cljc
index 48c379e366e..e8966fca0f7 100644
--- a/src/metabase/lib/equality.cljc
+++ b/src/metabase/lib/equality.cljc
@@ -104,46 +104,70 @@
(sequential? x) ((get-method = :dispatch-type/sequential) x y)
:else (clojure.core/= x y)))
-(defn- columns-equal-by-fn-when-non-nil-in-both [f col-1 col-2]
+;; for debugging purposes the implementation of [[=]] for two columns return the "reason" things are not equal so we
+;; can log this; [[=]] is basically just (not <reason>)
+
+(defn- columns-not-equal-by-fn-when-non-nil-in-both
+ [f col-1 col-2]
(let [v1 (f col-1)
v2 (f col-2)]
- (if (and v1 v2)
- (clojure.core/= v1 v2)
- true)))
+ (when (and v1 v2 (not= v1 v2))
+ f)))
-(defn- columns-equal-by-fn [f col-1 col-2]
- (clojure.core/= (f col-1) (f col-2)))
+(defn- columns-not-equal-by-fn
+ [f col-1 col-2]
+ (when (not= (f col-1) (f col-2))
+ f))
(defn- ignore-default-temporal-bucket [bucket]
(when-not (clojure.core/= bucket :default)
bucket))
-(defmethod = :metadata/column
+(defn- columns-not-equal-reason
+ "Returns a keyword representing the reason why two columns fail an [[=]] check (for debugging purposes)."
[col-1 col-2]
- (and
- ;; two column metadatas with different IDs are NEVER equal.
- (columns-equal-by-fn-when-non-nil-in-both :id col-1 col-2)
- ;; from the same source.
- (columns-equal-by-fn-when-non-nil-in-both :lib/source col-1 col-2)
- ;; same join alias
- (columns-equal-by-fn lib.join.util/current-join-alias col-1 col-2)
- ;;
- ;; columns that don't have the same binning or temporal bucketing are never the same.
- ;;
- ;; same binning
- (columns-equal-by-fn :metabase.lib.field/binning col-1 col-2)
- ;; same bucketing
- (columns-equal-by-fn (comp ignore-default-temporal-bucket lib.temporal-bucket/raw-temporal-bucket) col-1 col-2)
- ;; check `:inherited-temporal-unit` as well if both columns have it.
- (columns-equal-by-fn-when-non-nil-in-both (comp ignore-default-temporal-bucket :inherited-temporal-unit) col-1 col-2)
- ;; finally make sure they have the same `:lib/source-column-alias` (if both columns have it) or `:name` (if for
- ;; some reason they do not)
+ (or
+ ;; two column metadatas with different IDs are NEVER equal.
+ (columns-not-equal-by-fn-when-non-nil-in-both :id col-1 col-2)
+ ;; from the same source.
+ (columns-not-equal-by-fn-when-non-nil-in-both :lib/source col-1 col-2)
+ ;; same join alias
+ (columns-not-equal-by-fn :metabase.lib.join/join-alias col-1 col-2)
+ ;; same FK Field (for implicitly joined columns)
+ (columns-not-equal-by-fn-when-non-nil-in-both :fk-field-id col-1 col-2)
+ (columns-not-equal-by-fn :fk-join-alias col-1 col-2)
+ ;; TODO (Cam 9/4/25) -- not super clear that this ought to be a reason for columns to be considered different since
+ ;; `:fk-field-name` doesn't really seem to be super important... but this check seems to be needed, otherwise when
+ ;; a there are multiple remappings from Col A => Col B (e.g. in a self-join) we'll potentially accidentally match
+ ;; the wrong one. Maybe we can figure out a better way to make sure that doesn't happen.
+ (columns-not-equal-by-fn-when-non-nil-in-both :fk-field-name col-1 col-2)
+ ;;
+ ;; columns that don't have the same binning or temporal bucketing are never the same.
+ ;;
+ ;; same binning
+ (columns-not-equal-by-fn :metabase.lib.field/binning col-1 col-2)
+ ;; same bucketing
+ (when (columns-not-equal-by-fn (comp ignore-default-temporal-bucket lib.temporal-bucket/raw-temporal-bucket) col-1 col-2)
+ 'temporal-bucket)
+ ;; check `:inherited-temporal-unit` as well if both columns have it.
+ (when (columns-not-equal-by-fn-when-non-nil-in-both (comp ignore-default-temporal-bucket :inherited-temporal-unit) col-1 col-2)
+ :inherited-temporal-unit)
+ ;; finally make sure they have the same `:lib/source-column-alias` (if both columns have it) or `:name` (if for
+ ;; some reason they do not)
(let [k (m/find-first (fn [k]
(and (k col-1)
(k col-2)))
[:lib/source-column-alias :name])]
(assert k "No key common to both columns")
- (columns-equal-by-fn k col-1 col-2))))
+ (columns-not-equal-by-fn k col-1 col-2))))
+
+(defmethod = :metadata/column
+ [col-1 col-2]
+ (let [not-equal-reason (columns-not-equal-reason col-1 col-2)]
+ (if not-equal-reason
+ (log/debugf "Columns are not equal. Reason: %s" (pr-str not-equal-reason))
+ (log/debug "Columns are equal."))
+ (not not-equal-reason)))
(mu/defn- resolve-field-id-in-source-card :- ::lib.schema.metadata/column
"Integer Field ID: get metadata from the metadata provider. If this is the first stage of the query, merge in
diff --git a/src/metabase/lib/field/resolution.cljc b/src/metabase/lib/field/resolution.cljc
index 729c92998da..461664b30b7 100644
--- a/src/metabase/lib/field/resolution.cljc
+++ b/src/metabase/lib/field/resolution.cljc
@@ -608,6 +608,29 @@
(-> (lib.expression/expression-metadata query stage-number expr)
(assoc :lib/source-column-alias id-or-name))))))
+(declare resolve-from-previous-stage-or-source)
+
+(defn- resolve-nonexistent-deduplicated-column-name
+ "Resolve a ref like `CATEGORY_2` to `CATEGORY` if the query only has the latter."
+ [query stage-number id-or-name]
+ (when (string? id-or-name)
+ (when-let [[_match original-name suffix] (re-matches #"^(\w+)_([1-9][0-9]*)$" id-or-name)]
+ (let [suffix (parse-long suffix)
+ new-suffix (dec suffix)
+ ;; e.g. `CATEGORY_3` becomes `CATEGORY_2`; `CATEGORY_2` becomes `CATEGORY`
+ new-name (if (<= new-suffix 1)
+ original-name
+ (str original-name \_ new-suffix))]
+ (log/debugf "Failed to resolve %s, trying to resolve Field %s instead..." (pr-str id-or-name) (pr-str new-name))
+ (let [resolved (resolve-from-previous-stage-or-source query stage-number new-name)]
+ (if (::fallback-metadata? resolved)
+ (do
+ (log/debugf "Failed to resolve %s as %s" (pr-str id-or-name) (pr-str new-name))
+ nil)
+ (do
+ (log/debugf "Successfully resolved %s as %s" (pr-str id-or-name) (pr-str new-name))
+ resolved)))))))
+
(mu/defn- resolve-from-previous-stage-or-source :- ::lib.metadata.calculation/visible-column
[query :- ::lib.schema/query
stage-number :- :int
@@ -624,6 +647,9 @@
;; try looking in the expressions in this stage to see if someone incorrectly used a field ref for an
;; expression.
(maybe-resolve-expression-in-current-stage query stage-number id-or-name)
+ ;; if that fails and this is a deduplicated name like `CATEGORY_2` then try looking for `CATEGORY` and
+ ;; so forth
+ (resolve-nonexistent-deduplicated-column-name query stage-number id-or-name)
;; if we STILL can't find a match, return made-up fallback metadata.
(fallback-metadata id-or-name))]
(when col
diff --git a/src/metabase/lib/walk.cljc b/src/metabase/lib/walk.cljc
index 795906def8e..9e4b3580ce3 100644
--- a/src/metabase/lib/walk.cljc
+++ b/src/metabase/lib/walk.cljc
@@ -385,6 +385,21 @@
clauses
(range (count clauses)))))
+(mu/defn walk-clauses-in-stage :- [:maybe ::lib.schema/stage.mbql]
+ "Walk all the clauses in a stage. Calls
+
+ (f clause)
+
+ for every clause in the stage."
+ [stage :- ::lib.schema/stage
+ f :- [:=> [:cat :any] :any]]
+ (when (lib.util/mbql-stage? stage)
+ (reduce
+ (fn [stage k]
+ (m/update-existing stage k walk-clauses* f))
+ stage
+ [:aggregation :breakout :expressions :fields :filters :order-by])))
+
(mu/defn walk-clauses :- ::lib.schema/query
"Walk all the MBQL clauses in a query in a depth-first manner.
@@ -420,9 +435,4 @@
(walk-clauses* f)))))
:lib.walk/stage
- (when (lib.util/mbql-stage? stage-or-join)
- (reduce
- (fn [stage k]
- (m/update-existing stage k walk-clauses* f))
- stage-or-join
- [:aggregation :breakout :expressions :fields :filters :order-by])))))))
+ (walk-clauses-in-stage stage-or-join f))))))
diff --git a/src/metabase/query_processor/util/nest_query.clj b/src/metabase/query_processor/util/nest_query.clj
index 0c2f479c3bf..7f8869b4d61 100644
--- a/src/metabase/query_processor/util/nest_query.clj
+++ b/src/metabase/query_processor/util/nest_query.clj
@@ -1,352 +1,140 @@
(ns metabase.query-processor.util.nest-query
- "Utility functions for raising/nesting parts of MBQL queries. Currently, this only has [[nest-expressions]], but in
- the future hopefully we can generalize this a bit so we can do more things that require us to introduce another
- level of nesting, e.g. support window functions.
+ "The [[nest-expressions]] query transformation (see docstring for more info).
- (This namespace is here rather than in the shared MBQL lib because it relies on other QP-land utils like the QP
- refs stuff.)"
- (:refer-clojure :exclude [mapv select-keys some])
+ TODO (Cam 10/22/25) -- this is a pure-MBQL-5 high-level query transformation, and almost certainly belongs in Lib
+ rather than in QP -- we should move it there. (This also applies
+ to [[metabase.query-processor.util.transformations.nest-breakouts]])."
+ (:refer-clojure :exclude [select-keys some])
(:require
[medley.core :as m]
- [metabase.api.common :as api]
- ;; legacy usage -- don't use Legacy MBQL utils in QP code going forward, prefer Lib. This will be updated to use
- ;; Lib only soon
- ^{:clj-kondo/ignore [:discouraged-namespace]}
- [metabase.legacy-mbql.schema :as mbql.s]
- ^{:clj-kondo/ignore [:discouraged-namespace :deprecated-namespace]}
- [metabase.legacy-mbql.util :as mbql.u]
[metabase.lib.core :as lib]
- [metabase.lib.metadata :as lib.metadata]
- [metabase.lib.schema.common :as lib.schema.common]
+ [metabase.lib.equality :as lib.equality]
+ [metabase.lib.schema :as lib.schema]
[metabase.lib.util.match :as lib.util.match]
- [metabase.query-processor.middleware.annotate.legacy-helper-fns :as annotate.legacy-helper-fns]
- ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store]
- [metabase.query-processor.util.add-alias-info :as add]
+ [metabase.lib.walk :as lib.walk]
[metabase.util :as u]
- [metabase.util.log :as log]
[metabase.util.malli :as mu]
- [metabase.util.performance :as perf :refer [mapv select-keys some]]))
-
-(defn- all-fields-for-table [table-id]
- (->> (lib.metadata/fields (qp.store/metadata-provider) table-id)
- ;; The remove line is taken from the add-implicit-clauses middleware. It shouldn't be necessary, because any
- ;; unused fields should be dropped from the inner query. It also shouldn't hurt anything, because the outer
- ;; query shouldn't use these fields in the first place.
- ;;
- ;; TODO (Cam 9/12/25) -- this shouldn't be necessary anymore
- (remove #(#{:sensitive :retired} (:visibility-type %)))
- (map (fn [field]
- [:field (u/the-id field) nil]))))
-
-(defn- add-all-fields
- "This adds all non-sensitive/retired fields (including fields with parent ids) to the passed-in query.
-
- The issue is that nest-query mostly relies on the preprocessor (specifically, the add-implicit-clauses middleware)
- to add all possible fields to the newly created inner query. However, add-implicit-fields ignores fields with a
- parent id. The main user of parent-id is mongo, and mongo doesn't use nest-query, so this usually isn't an
- issue. However, bigquery also uses parent-id, and bigquery is a sql driver that uses nest-query. As a result,
- without this function, nest-query wasn't adding bigquery struct member fields to the inner query, which caused sql
- errors. This function adds in any missing fields and fixes those errors."
- [{{source-table-id :source-table, :keys [fields]} :query, :as outer-query}]
- (if source-table-id
- (let [all-fields (all-fields-for-table source-table-id)
- existing-fields (into #{} (map second) fields)]
- (assoc-in outer-query [:query :fields]
- (into fields
- (remove (comp existing-fields second))
- all-fields)))
- outer-query))
-
-(defn- remove-namespaced-options [options]
- (when options
- (not-empty (into {}
- (remove (fn [[k _]]
- (qualified-keyword? k)))
- options))))
-
-;;; TODO (Cam 6/16/25) -- duplicated with [[metabase.query-processor.middleware.resolve-joins/normalize-clause]], but
-;;; this doesn't matter much since we will rewrite this whole namespace in MLv2 soon.
-(defn- normalize-clause
- "Normalize a `:field`/`:expression`/`:aggregation` clause by removing extra info so it can serve as a key for
- `:qp/refs`. This removes `:source-field` if it is present -- don't use the output of this for anything but internal
- key/distinct comparison purposes."
- [clause]
- (lib.util.match/match-one clause
- ;; optimization: don't need to rewrite a `:field` clause without any options
- [:field _ nil]
- &match
-
- [:field id-or-name opts]
- ;; this doesn't use [[mbql.u/update-field-options]] because this gets called a lot and the overhead actually adds up
- ;; a bit
- [:field id-or-name (remove-namespaced-options (cond-> (dissoc opts :source-field :effective-type)
- (integer? id-or-name) (dissoc :base-type)))]
-
- ;; for `:expression` and `:aggregation` references, remove the options map if they are empty.
- [:expression expression-name opts]
- (if-let [opts (remove-namespaced-options opts)]
- [:expression expression-name opts]
- [:expression expression-name])
-
- [:aggregation index opts]
- (if-let [opts (remove-namespaced-options opts)]
- [:aggregation index opts]
- [:aggregation index])
-
- _
- &match))
-
-(defn- joined-fields [inner-query]
- (m/distinct-by
- normalize-clause
- (lib.util.match/match (perf/prewalk (fn [x]
- (if (map? x)
- (dissoc x :source-query :source-metadata :temporal-unit)
- x))
- inner-query)
- [:field _ (_ :guard :join-alias)]
- &match)))
-
-(defn- keep-source+alias-props [field]
- (update field 2 select-keys [::add/source-alias ::add/source-table :join-alias]))
-
-(defn- nfc-root [[_ field-id]]
- (when-let [field (and (int? field-id)
- (lib.metadata/field (qp.store/metadata-provider) field-id))]
- (when-let [nfc-root (first (:nfc-path field))]
- {:table_id (:table-id field)
- :name nfc-root})))
-
-(defn- field-id-props [[_ field-id]]
- (when-let [field (and (int? field-id)
- (lib.metadata/field (qp.store/metadata-provider) field-id))]
- {:table_id (:table-id field)
- :name (:name field)}))
-
-(defn- ->nominal-ref
- "Transforms a ref into a simplified version of the form it would take as a nominal ref in a later stage.
-
- Nominal refs use the `desired-alias` as its `id-or-name`, and have `::add/source-table ::add/source` in the options.
-
- Other options (`:position`, `:join-alias`, other aliases) are dropped, since those are internal to the stage where the
- column joined the party, not to later stages.
-
- Refs which are either missing `::add/desired-alias`, or which are coming directly from a table or join, are skipped.
- We want to pick up the usages only, not anything from `:fields` lists."
- [[_tag _id-or-name {::add/keys [source-alias source-table]}]]
- (when (and source-alias
- (= source-table ::add/source))
- [:field source-alias {::add/source-table ::add/source}]))
-
-(defn- remove-unused-fields [inner-query source]
- (let [usages (lib.util.match/match inner-query #{:field :expression})
- used-refs (into #{} (map keep-source+alias-props) usages)
- nominal-refs (into #{} (keep ->nominal-ref) usages)
- nfc-roots (into #{} (keep nfc-root) used-refs)]
- (letfn [(used? [[_tag _id-or-name {::add/keys [source-table]}, :as a-ref]]
- (or (contains? used-refs (keep-source+alias-props a-ref))
- ;; We should also consider a Field to be used if we're referring to it with a nominal field literal
- ;; ref in the next stage -- that's actually how you're supposed to be doing it anyway.
- (and (= source-table ::add/source)
- (contains? nominal-refs (->nominal-ref a-ref)))
- (contains? nfc-roots (field-id-props a-ref))))
- (used?* [a-ref]
- (u/prog1 (used? a-ref)
- (if <>
- (log/debugf "Keeping used ref:\n%s" (u/pprint-to-str a-ref))
- (log/debugf "Removing unused ref:\n%s" (u/pprint-to-str (keep-source+alias-props a-ref))))))
- (remove-unused [refs]
- (into []
- (comp (filter used?*)
- (m/distinct-by (fn [[_tag _id-or-name opts, :as _a-ref]]
- (or (::add/desired-alias opts)
- ;; if the field ref doesn't have a desired alias (probably because we
- ;; added it in this namespace and haven't generated yet, then always
- ;; keep it... just use a random UUID that should always be distinct
- (random-uuid)))))
- refs))]
- (update source :fields remove-unused))))
-
-(defn- append-join-fields
- "This (supposedly) matches the behavior of [[metabase.lib.stage/add-cols-from-join]]. When we migrate this namespace
- to Lib we can maybe use that."
- [fields join-fields]
- ;; we shouldn't consider different type info to mean two Fields are different even if everything else is the same. So
- ;; give everything `:base-type` of `:type/*` (it will complain if we remove `:base-type` entirely from fields with a
- ;; string name)
- (letfn [(opts-signature [opts]
- (not-empty
- (merge
- (u/select-non-nil-keys opts [:join-alias :binning])
- ;; for purposes of deduplicating stuff, temporal unit = default is the same as not specifying temporal
- ;; unit at all. Should that be part of normalization? Maybe, but there is some logic around adding default
- ;; temporal bucketing that we don't do if `:default` is explicitly specified.
- (when-let [temporal-unit (:temporal-unit opts)]
- (when-not (= temporal-unit :default)
- {:temporal-unit temporal-unit})))))
- (ref-signature [[tag id-or-name opts, :as _ref]]
- [tag id-or-name (opts-signature opts)])]
- (into []
- (comp cat
- (m/distinct-by ref-signature))
- [fields join-fields])))
-
-(defn- append-join-fields-to-fields
- "Add the fields from join `:fields`, if any, to the parent-level `:fields`."
- [inner-query join-fields]
- (cond-> inner-query
- (seq join-fields) (update :fields append-join-fields join-fields)))
-
-(mu/defn- nest-source :- ::mbql.s/SourceQuery
- [inner-query :- ::mbql.s/SourceQuery]
- (let [filter-clause (:filter inner-query)
- keep-filter? (and filter-clause
- (nil? (lib.util.match/match-one filter-clause :expression)))
- source (-> inner-query
- (select-keys [:source-table :source-query :source-metadata :joins :expressions])
- ;; preprocess this in a superuser context so it's not subject to permissions checks. To get here in
- ;; the first place we already had to do perms checks to make sure the query we're transforming is
- ;; itself ok, so we don't need to run another check.
- ;; (Not using mw.session/as-admin due to cyclic dependency.)
- (as-> $source (binding [api/*is-superuser?* true]
- ((requiring-resolve 'metabase.query-processor.preprocess/preprocess)
- {:database (u/the-id (lib.metadata/database (qp.store/metadata-provider)))
- :type :query
- :query $source})))
- lib/->legacy-MBQL
- add-all-fields
- add/add-alias-info
- :query
- (dissoc :limit)
- (append-join-fields-to-fields (joined-fields inner-query))
- (->> (remove-unused-fields inner-query))
- (cond-> keep-filter?
- (assoc :filter filter-clause)))]
- (-> inner-query
- (dissoc :source-table :source-metadata :joins)
- (assoc :source-query source)
- (cond-> keep-filter? (dissoc :filter)))))
-
-(mu/defn- infer-expression-type :- [:maybe ::lib.schema.common/base-type]
- [inner-query :- :map
- expression :- [:maybe ::mbql.s/FieldOrExpressionDef]]
- (when expression
- #_{:clj-kondo/ignore [:deprecated-var]}
- (let [mlv2-query (annotate.legacy-helper-fns/legacy-inner-query->mlv2-query inner-query)]
- (lib/type-of mlv2-query (lib/->pMBQL expression)))))
-
-(defn- raise-source-query-expression-ref
- "Convert an `:expression` reference from a source query into an appropriate `:field` clause for use in the surrounding
- query."
- [{:keys [source-query], :as query} [_ expression-name opts :as _clause]]
- (let [expression-definition #_{:clj-kondo/ignore [:deprecated-var]} (mbql.u/expression-with-name query expression-name)
- base-type (infer-expression-type query expression-definition)
- {::add/keys [desired-alias]} (lib.util.match/match-one source-query
- [:expression (_ :guard (partial = expression-name)) source-opts]
- source-opts)
- source-alias (or desired-alias expression-name)]
- [:field
- source-alias
- (-> opts
- (assoc :base-type (or base-type :type/*)
- ::add/source-table ::add/source
- ::add/source-alias source-alias))]))
-
-(defn- coerced-field?
- [field-id]
- (contains? (lib.metadata/field (qp.store/metadata-provider) field-id) :coercion-strategy))
-
-(defn- coercible-field-ref?
- [form]
- (and (vector? form)
- (let [[tag id-or-name opts] form]
- (and (= tag :field)
- (not (:qp/ignore-coercion opts))
- (or (contains? opts :temporal-unit)
- (and (int? id-or-name)
- (coerced-field? id-or-name)))))))
-
-(defn- rewrite-fields-and-expressions [query]
- (lib.util.match/replace query
- ;; don't rewrite anything inside any source queries or source metadata.
- (_ :guard (constantly (some (partial contains? (set &parents))
- [:source-query :source-metadata])))
- &match
-
- :expression
- (raise-source-query-expression-ref query &match)
-
- ;; Mark all Fields at the new top level as `:qp/ignore-coercion` so QP implementations know not to apply coercion
- ;; or whatever to them a second time.
- ;; In fact, we don't mark all Fields, only the ones we deem coercible. Marking all would make a bunch of tests
- ;; fail, but it might still make sense. For example, #48721 would have been avoided by unconditional marking.
- (_ :guard coercible-field-ref?)
- (recur #_{:clj-kondo/ignore [:deprecated-var]}
- (mbql.u/update-field-options &match assoc :qp/ignore-coercion true))
-
- [:field id-or-name (opts :guard :join-alias)]
- (let [{::add/keys [desired-alias]} (lib.util.match/match-one (:source-query query)
- [:field
- (_ :guard (partial = id-or-name))
- (matching-opts :guard #(= (:join-alias %) (:join-alias opts)))]
- matching-opts)]
- [:field id-or-name (cond-> opts
- desired-alias (assoc ::add/source-alias desired-alias
- ::add/desired-alias desired-alias))])
-
- ;; Some refs in the outer stage might be referring to these fields by `:name` (eg. ID) and not
- ;; properly by their `desired-alias`/this ref's `source-alias` (eg. "People - User__ID")
- ;; Since these refs are across stages, there's no need to adjust `:expression` refs.
- [:field (id-or-name :guard string?) (opts :guard ::add/source-alias)]
- [:field (::add/source-alias opts) opts]
-
- ;; when recursing into joins use the refs from the parent level.
- (m :guard (every-pred map? :joins))
- (let [{:keys [joins]} m]
- (-> (dissoc m :joins)
- rewrite-fields-and-expressions
- (assoc :joins (mapv (fn [join]
- (assoc join :qp/refs (:qp/refs query)))
- joins))))))
-
-(defn- should-nest-expressions?
- "Whether we should nest the expressions in a inner query; true if
-
- 1. there are some expression definitions in the inner query, AND
-
- 2. there are some breakouts OR aggregations OR order-by in the inner query
-
- 3. AND the breakouts/aggregations/order-bys contain at least one `:expression` reference."
- [{:keys [expressions], breakouts :breakout, aggregations :aggregation, order-bys :order-by, :as _inner-query}]
- (and
- ;; 1. has some expression definitions
- (seq expressions)
- ;; 2. has some breakouts or aggregations or order-by
- (or (seq breakouts)
- (seq aggregations)
- (seq order-bys))
- ;; 3. contains an `:expression` ref
- (lib.util.match/match-one (concat breakouts aggregations order-bys)
- :expression)))
-
-(mu/defn- nest-expressions* :- ::mbql.s/SourceQuery
- [inner-query :- ::mbql.s/SourceQuery]
- (let [{:keys [expressions]
- :as inner-query} (m/update-existing inner-query :source-query nest-expressions*)]
- (if-not (should-nest-expressions? inner-query)
- inner-query
- (let [{:keys [source-query], :as inner-query} (nest-source inner-query)
- inner-query (rewrite-fields-and-expressions inner-query)
- source-query (assoc source-query :expressions expressions)]
- (-> inner-query
- (dissoc :source-query :expressions :expression-idents)
- (assoc :source-query source-query))))))
-
-(mu/defn nest-expressions :- ::mbql.s/SourceQuery
- "Pushes the `:source-table`/`:source-query`, `:expressions`, and `:joins` in the top-level of the query into a
- `:source-query` and updates `:expression` references and `:field` clauses with `:join-alias`es accordingly. See
- tests for examples. This is used by the SQL QP to make sure expressions happen in a subselect."
- [inner-query :- ::mbql.s/SourceQuery]
- (-> inner-query
- nest-expressions*
- add/add-alias-info))
+ [metabase.util.performance :refer [select-keys some]]))
+
+;; Mark all Fields at the new top level as `:qp/ignore-coercion` so QP implementations know not to apply coercion
+;; or whatever to them a second time.
+;; In fact, we don't mark all Fields, only the ones we deem coercible. Marking all would make a bunch of tests
+;; fail, but it might still make sense. For example, #48721 would have been avoided by unconditional marking.
+
+(defn- contains-expression-refs? [location]
+ (lib.util.match/match-one location :expression))
+
+(defn- should-nest-expressions? [query path]
+ (and (lib.walk/apply-f-for-stage-at-path lib/mbql-stage? query path)
+ (some (fn [f]
+ (contains-expression-refs? (lib.walk/apply-f-for-stage-at-path f query path)))
+ [lib/breakouts
+ lib/aggregations
+ lib/order-bys])))
+
+(def ^:private first-stage-keys
+ #{:expressions :joins :source-table :source-card :filters})
+
+;; TODO (Cam 10/22/25) -- somewhat duplicated with/copied
+;; from [[metabase.query-processor.util.transformations.nest-breakouts/fields-used-in-breakouts-aggregations-or-expressions]]
+(defn- fields-needed-by-second-stage [query path stage]
+ (let [stage' (apply dissoc stage first-stage-keys)
+ refs (volatile! (transient []))]
+ ;; temporarily disable enforcement since this stage is technically invalid since we removed all
+ ;; the [[first-stage-keys]]
+ (binding [lib.schema/*HACK-disable-ref-validation* true]
+ (lib.walk/walk-clauses-in-stage
+ stage'
+ (fn [clause]
+ (u/prog1 clause
+ (when (lib/clause-of-type? clause #{:field :expression})
+ ;; preserve the unbucketed/unbinned versions of things in the new first stage; we'll apply the
+ ;; binning/bucketing in the second stage instead
+ (let [unbucketed-ref (-> clause
+ (cond-> (lib/clause-of-type? clause :field) (lib/with-binning nil))
+ (lib/with-temporal-bucket nil)
+ ;; unset these just to be safe in case they were set -- they're technically
+ ;; allowed if the binning/bucketing is happening at this stage but they're no
+ ;; longer happening here so leaving them in place would be incorrect.
+ (lib/update-options dissoc
+ :metabase.lib.field/original-temporal-unit
+ :original-temporal-unit
+ :lib/original-binning))]
+ (vswap! refs conj! unbucketed-ref)))))))
+ (into
+ []
+ (comp (m/distinct-by (fn [[tag opts id-or-name]]
+ [tag
+ (select-keys opts [:join-alias])
+ id-or-name]))
+ ;; if you are a psycho and have both a Field ID ref and a Field name ref to the same column the we need to
+ ;; deduplicate those otherwise this is going to result in queries with duplicate column
+ ;; names (see [[literal-boolean-expressions-and-fields-in-conditions-test]])
+ (m/distinct-by (fn [a-ref]
+ (-> (lib.walk/apply-f-for-stage-at-path lib/metadata query path a-ref)
+ ((juxt :lib/source-column-alias lib/current-join-alias))))))
+ (persistent! @refs))))
+
+(defn- new-first-stage [query path stage]
+ (-> stage
+ (select-keys (conj first-stage-keys :lib/type))
+ (assoc :fields (lib/fresh-uuids (fields-needed-by-second-stage query path stage)))))
+
+(defn- new-second-stage [query path stage]
+ (let [returned-cols (lib.walk/apply-f-for-stage-at-path lib/returned-columns query path)]
+ (letfn [(update-ref [a-ref]
+ (let [col (lib.walk/apply-f-for-stage-at-path lib/metadata query path a-ref)
+ unbucketed-col (-> col
+ (lib/with-binning nil)
+ (lib/with-temporal-bucket nil))
+ returned-col (m/find-first (partial lib.equality/= unbucketed-col) returned-cols)]
+ (-> returned-col
+ lib/update-keys-for-col-from-previous-stage
+ (lib/with-binning (lib/binning col))
+ (lib/with-temporal-bucket (lib/raw-temporal-bucket col))
+ lib/ref)))]
+ ;; temporarily disable enforcement since this stage will be invalid while we're messing with it... it will look
+ ;; pretty nice when we're done tho.
+ (binding [lib.schema/*HACK-disable-ref-validation* true]
+ (-> stage
+ (as-> $stage (apply dissoc $stage first-stage-keys))
+ (lib.walk/walk-clauses-in-stage
+ (fn [clause]
+ (cond-> clause
+ (lib/clause-of-type? clause #{:field :expression})
+ update-ref))))))))
+
+(mu/defn- nest-expressions* :- [:sequential {:min 2, :max 2} ::lib.schema/stage.mbql]
+ [query :- ::lib.schema/query
+ path :- ::lib.walk/path
+ stage :- ::lib.schema/stage.mbql]
+ (let [new-first-stage (new-first-stage query path stage)]
+ [new-first-stage
+ (new-second-stage (assoc-in query path new-first-stage) path stage)]))
+
+(mu/defn nest-expressions :- ::lib.schema/query
+ "For queries with `expressions` in the final stage, adds an additional stage and moves the breakouts, aggregations,
+ and order bys into the new final stage. This is because lots of our SQL databases don't really like when you do
+ stuff like this:
+
+ SELECT (x + ?) AS x_1
+ FROM my_table
+ GROUP BY (x + ?)
+ ORDER BY (x + ?) ASC
+
+ Why? They are dumb and can't figure out `x + ?` is the same thing. So instead we will introduce an additional stage
+ that will give us SQL that looks like this:
+
+ SELECT x_1 AS x_1
+ FROM (
+ SELECT (x + ?) AS x_1
+ FROM my_table
+ ) source
+ GROUP BY x_1
+ ORDER BY x_1 ASC"
+ [query :- ::lib.schema/query]
+ (lib.walk/walk-stages
+ query
+ (fn [query path stage]
+ (when (should-nest-expressions? query path)
+ (nest-expressions* query path stage)))))
diff --git a/src/metabase/query_processor/util/transformations/nest_breakouts.clj b/src/metabase/query_processor/util/transformations/nest_breakouts.clj
index 07b427b348a..86b485d2355 100644
--- a/src/metabase/query_processor/util/transformations/nest_breakouts.clj
+++ b/src/metabase/query_processor/util/transformations/nest_breakouts.clj
@@ -1,6 +1,6 @@
(ns metabase.query-processor.util.transformations.nest-breakouts
"TODO (Cam 8/7/25) -- this is a pure-MBQL-5 high-level query transformation, and almost certainly belongs in Lib
- rather than in QP -- we should move it there."
+ rather than in QP -- we should move it there. (This also applies to [[metabase.query-processor.util.nest-query]])."
(:refer-clojure :exclude [mapv select-keys some])
(:require
[flatland.ordered.set :as ordered-set]
@@ -71,23 +71,23 @@
(def ^:private granularity
{:time-unbucketed 0
- :minute 1
- :minute-of-hour 2
- :hour 3
- :hour-of-day 4
- :day 5
+ :minute 1
+ :minute-of-hour 2
+ :hour 3
+ :hour-of-day 4
+ :day 5
:date-unbucketed 5
- :day-of-week 6
- :day-of-month 7
- :day-of-year 8
- :week 9
- :week-of-year 10
- :month 11
- :month-of-year 12
- :quarter 13
+ :day-of-week 6
+ :day-of-month 7
+ :day-of-year 8
+ :week 9
+ :week-of-year 10
+ :month 11
+ :month-of-year 12
+ :quarter 13
:quarter-of-year 14
- :year 15
- :year-of-era 16})
+ :year 15
+ :year-of-era 16})
(defn- original-temporal-unit
[temporal-attributes]
diff --git a/src/metabase/xrays/automagic_dashboards/combination.clj b/src/metabase/xrays/automagic_dashboards/combination.clj
index d8673ae110f..fdb7b704685 100644
--- a/src/metabase/xrays/automagic_dashboards/combination.clj
+++ b/src/metabase/xrays/automagic_dashboards/combination.clj
@@ -88,23 +88,13 @@
(let [model? (and (mi/instance-of? :model/Card source)
(queries/model? source))
mp (lib-be/application-database-metadata-provider database)
- query (cond-> (lib/query mp (cond
- (mi/instance-of? :model/Table source) (lib.metadata/table mp (u/the-id source))
- (mi/instance-of? :model/Card source) (lib.metadata/card mp (u/the-id source))))
- (:xrays/aggregations metric-definition)
- (as-> $query (reduce lib/aggregate $query (:xrays/aggregations metric-definition)))
-
- (:xrays/breakouts metric-definition)
- (as-> $query (reduce lib/breakout $query (:xrays/breakouts metric-definition)))
-
- (:xrays/filters metric-definition)
- (as-> $query (reduce lib/filter $query (:xrays/filters metric-definition)))
-
- (not model?)
- (as-> $query (reduce
- lib/filter
- $query
- query-filter)))]
+ query (as-> (lib/query mp (cond
+ (mi/instance-of? :model/Table source) (lib.metadata/table mp (u/the-id source))
+ (mi/instance-of? :model/Card source) (lib.metadata/card mp (u/the-id source)))) $query
+ (reduce lib/aggregate $query (:xrays/aggregations metric-definition))
+ (reduce lib/breakout $query (:xrays/breakouts metric-definition))
+ (reduce lib/filter $query (:xrays/filters metric-definition))
+ (reduce lib/filter $query (when (not model?) query-filter)))]
(assoc ground-metric-with-dimensions :dataset_query query)))
(defn- instantiate-visualization
diff --git a/src/metabase/xrays/automagic_dashboards/comparison.clj b/src/metabase/xrays/automagic_dashboards/comparison.clj
index 08bd000a048..cd64c20159c 100644
--- a/src/metabase/xrays/automagic_dashboards/comparison.clj
+++ b/src/metabase/xrays/automagic_dashboards/comparison.clj
@@ -250,6 +250,10 @@
"Create a comparison dashboard based on dashboard `dashboard` comparing subsets of
the dataset defined by segments `left` and `right`."
[dashboard left right opts]
+ ;; disable ref validation because X-Rays does stuff in a wacko manner, it adds a bunch of filters and whatever that
+ ;; use columns from joins before adding the joins themselves (same with expressions), which is technically invalid
+ ;; at the time it happens but ends up resulting in a valid query at the end of the day. Maybe one day we can rework
+ ;; this code to be saner
(binding [lib.schema/*HACK-disable-ref-validation* true]
(let [opts (-> opts
(m/update-existing-in [:left :cell-query] (partial lib/normalize ::ads/root.cell-query))
diff --git a/src/metabase/xrays/automagic_dashboards/core.clj b/src/metabase/xrays/automagic_dashboards/core.clj
index 0808f4048a7..919c169314c 100644
--- a/src/metabase/xrays/automagic_dashboards/core.clj
+++ b/src/metabase/xrays/automagic_dashboards/core.clj
@@ -831,6 +831,10 @@
entity
getter-fn
setter-fn]
+ ;; disable ref validation because X-Rays does stuff in a wacko manner, it adds a bunch of filters and whatever that
+ ;; use columns from joins before adding the joins themselves (same with expressions), which is technically invalid
+ ;; at the time it happens but ends up resulting in a valid query at the end of the day. Maybe one day we can rework
+ ;; this code to be saner
(binding [lib.schema/*HACK-disable-ref-validation* true]
(if-let [element-value (some-> entity :dataset_query not-empty getter-fn)]
(letfn [(splice-element [dashcard]
@@ -905,6 +909,10 @@
cell-url (format "%sadhoc/%s/cell/%s" public-endpoint
(magic.util/encode-base64-json (:dataset_query query))
(magic.util/encode-base64-json cell-query))]
+ ;; disable ref validation because X-Rays does stuff in a wacko manner, it adds a bunch of filters and whatever
+ ;; that use columns from joins before adding the joins themselves (same with expressions), which is technically
+ ;; invalid at the time it happens but ends up resulting in a valid query at the end of the day. Maybe one day we
+ ;; can rework this code to be saner
(binding [lib.schema/*HACK-disable-ref-validation* true]
(query-based-analysis root opts
(when cell-query
diff --git a/src/metabase/xrays/automagic_dashboards/populate.clj b/src/metabase/xrays/automagic_dashboards/populate.clj
index 4fb6e62932b..00658a6be21 100644
--- a/src/metabase/xrays/automagic_dashboards/populate.clj
+++ b/src/metabase/xrays/automagic_dashboards/populate.clj
@@ -140,6 +140,10 @@
{query :dataset_query, :keys [title description width height id] :as dashcard} :- ::ads/card-template
[x y]]
(let [query-fields (when query
+ ;; disable ref validation because X-Rays does stuff in a wacko manner, it adds a bunch of
+ ;; filters and whatever that use columns from joins before adding the joins themselves (same
+ ;; with expressions), which is technically invalid at the time it happens but ends up resulting
+ ;; in a valid query at the end of the day. Maybe one day we can rework this code to be saner
(binding [lib.schema/*HACK-disable-ref-validation* true]
(-> {:dataset_query (lib-be/normalize-query query)}
queries/populate-card-query-fields
@@ -320,6 +324,10 @@
(mu/defn- create-dashboard-populate-dashcards :- [:sequential ::ads/dashcard]
[dashcards :- [:maybe [:sequential ::ads/card-template]]]
+ ;; disable ref validation because X-Rays does stuff in a wacko manner, it adds a bunch of filters and whatever that
+ ;; use columns from joins before adding the joins themselves (same with expressions), which is technically invalid
+ ;; at the time it happens but ends up resulting in a valid query at the end of the day. Maybe one day we can rework
+ ;; this code to be saner
(let [card-id->can-run-adhoc-query (binding [lib.schema/*HACK-disable-ref-validation* true]
(into {}
(map (juxt ::id :can_run_adhoc_query))
diff --git a/test/metabase/driver/sql/query_processor_test.clj b/test/metabase/driver/sql/query_processor_test.clj
index b6a9b955866..ab88253f084 100644
--- a/test/metabase/driver/sql/query_processor_test.clj
+++ b/test/metabase/driver/sql/query_processor_test.clj
@@ -15,6 +15,7 @@
[metabase.lib.test-util :as lib.tu]
[metabase.lib.test-util.macros :as lib.tu.macros]
[metabase.lib.test-util.notebook-helpers :as lib.tu.notebook]
+ [metabase.lib.test-util.places-cam-likes-metadata-provider :as lib.tu.places-cam-likes-metadata-provider]
[metabase.query-processor :as qp]
[metabase.query-processor.compile :as qp.compile]
[metabase.query-processor.middleware.limit :as limit]
@@ -459,21 +460,16 @@
DATE_TRUNC ("year" source.CREATED_AT) AS CREATED_AT
source.pivot-grouping AS pivot-grouping
COUNT (*) AS count]
- :from [{:select [ORDERS.USER_ID AS USER_ID
- ORDERS.PRODUCT_ID AS PRODUCT_ID
- ORDERS.CREATED_AT AS CREATED_AT
- ABS (0) AS pivot-grouping
- ;; TODO: The order here is not deterministic! It's coming
- ;; from [[metabase.query-processor.util.transformations.nest-breakouts]]
- ;; or [[metabase.query-processor.util.nest-query]], which walks the query looking
- ;; for refs in an arbitrary order, and returns `m/distinct-by` over that random
- ;; order. Changing the map keys on the inner query can perturb this order; if you
- ;; cause this test to fail based on shuffling the order of these joined fields
- ;; just edit the expectation to match the new order. Tech debt issue: #39396
- PRODUCTS__via__PRODUCT_ID.ID AS PRODUCTS__via__PRODUCT_ID__ID
- PEOPLE__via__USER_ID.ID AS PEOPLE__via__USER_ID__ID
- PRODUCTS__via__PRODUCT_ID.CATEGORY AS PRODUCTS__via__PRODUCT_ID__CATEGORY
- PEOPLE__via__USER_ID.SOURCE AS PEOPLE__via__USER_ID__SOURCE]
+ ;; TODO: The order here is not deterministic! It's coming
+ ;; from [[metabase.query-processor.util.transformations.nest-breakouts]]
+ ;; or [[metabase.query-processor.util.nest-query]], which walks the query looking for refs in an
+ ;; arbitrary order, and returns `m/distinct-by` over that random order. Changing the map keys on the
+ ;; inner query can perturb this order; if you cause this test to fail based on shuffling the order of
+ ;; these joined fields just edit the expectation to match the new order. Tech debt issue: #39396
+ :from [{:select [PRODUCTS__via__PRODUCT_ID.CATEGORY AS PRODUCTS__via__PRODUCT_ID__CATEGORY
+ PEOPLE__via__USER_ID.SOURCE AS PEOPLE__via__USER_ID__SOURCE
+ ORDERS.CREATED_AT AS CREATED_AT
+ ABS (0) AS pivot-grouping]
:from [ORDERS]
:left-join [{:select [PRODUCTS.ID AS ID
PRODUCTS.EAN AS EAN
@@ -819,14 +815,13 @@
(deftest ^:parallel expression-with-duplicate-column-name-test
(testing "Can we use expression with same column name as table (#14267)"
- (is (= '{:select [source.CATEGORY_2 AS CATEGORY_2
+ (is (= '{:select [source.CATEGORY AS CATEGORY
COUNT (*) AS count]
- :from [{:select [PRODUCTS.CATEGORY AS CATEGORY
- CONCAT (PRODUCTS.CATEGORY ?) AS CATEGORY_2]
+ :from [{:select [CONCAT (PRODUCTS.CATEGORY ?) AS CATEGORY]
:from [PRODUCTS]}
AS source]
- :group-by [source.CATEGORY_2]
- :order-by [source.CATEGORY_2 ASC]
+ :group-by [source.CATEGORY]
+ :order-by [source.CATEGORY ASC]
:limit [1]}
(-> (lib.tu.macros/mbql-query products
{:expressions {:CATEGORY [:concat $category "2"]}
@@ -1561,3 +1556,73 @@
:query
(->> (driver/prettify-native-form :h2))
str/split-lines))))))
+
+(deftest ^:parallel literal-boolean-expressions-and-fields-in-conditions-test
+ (testing "mixing Field ID refs and Field Name refs to the same column should not result in broken queries"
+ (let [true-value [:value true {:base_type :type/Boolean}]
+ false-value [:value false {:base_type :type/Boolean}]
+ mp lib.tu.places-cam-likes-metadata-provider/metadata-provider
+ query (lib/query
+ mp
+ {:database 1
+ :type :query
+ :query {:expressions {"T" true-value, "F" false-value}
+ :source-query {:source-table 1
+ :fields [[:field 2 nil]]}
+ :aggregation [[:count-where [:expression "T"]]
+ [:count-where [:expression "F"]]
+ ;; only a psycho would
+ [:count-where [:field "LIKED" {:base-type :type/Boolean}]]
+ [:count-where [:field 2 nil]]]
+ :filter [:or
+ [:field 2 nil]
+ [:field "LIKED" {:base-type :type/Boolean}]
+ [:expression "T"]]}})]
+ (is (= ["SELECT"
+ " SUM("
+ " CASE"
+ " WHEN \"source\".\"T\" THEN 1"
+ " ELSE 0.0"
+ " END"
+ " ) AS \"count_where\","
+ " SUM("
+ " CASE"
+ " WHEN \"source\".\"F\" THEN 1"
+ " ELSE 0.0"
+ " END"
+ " ) AS \"count_where_2\","
+ " SUM("
+ " CASE"
+ " WHEN \"source\".\"LIKED\" THEN 1"
+ " ELSE 0.0"
+ " END"
+ " ) AS \"count_where_3\","
+ " SUM("
+ " CASE"
+ " WHEN \"source\".\"LIKED\" THEN 1"
+ " ELSE 0.0"
+ " END"
+ " ) AS \"count_where_4\""
+ "FROM"
+ " ("
+ " SELECT"
+ " TRUE AS \"T\","
+ " FALSE AS \"F\","
+ " \"source\".\"LIKED\" AS \"LIKED\""
+ " FROM"
+ " ("
+ " SELECT"
+ " \"PUBLIC\".\"PLACES\".\"LIKED\" AS \"LIKED\""
+ " FROM"
+ " \"PUBLIC\".\"PLACES\""
+ " ) AS \"source\""
+ " WHERE"
+ " \"source\".\"LIKED\""
+ " OR \"source\".\"LIKED\""
+ " OR TRUE"
+ " ) AS \"source\""]
+ (-> query
+ qp.compile/compile
+ :query
+ (->> (driver/prettify-native-form :h2))
+ str/split-lines))))))
diff --git a/test/metabase/lib/field/resolution_test.cljc b/test/metabase/lib/field/resolution_test.cljc
index a0a480f3777..11af689a02f 100644
--- a/test/metabase/lib/field/resolution_test.cljc
+++ b/test/metabase/lib/field/resolution_test.cljc
@@ -1595,3 +1595,85 @@
:lib/uuid "00000000-0000-0000-0000-000000000000"
:join-alias "Q1"}
"O__ID"])))))
+
+(deftest ^:parallel fallback-resolve-deduplicated-column-name-to-undeduplicated-name-test
+ ;; see https://metaboat.slack.com/archives/C0645JP1W81/p1761241427398479 for more context
+ (testing "If a query has something like `CATEGORY` but no `CATEGORY_2` we should resolve a `CATEGORY_2` ref to `CATEGORY`"
+ (let [query (lib/query
+ meta/metadata-provider
+ {:lib/type :mbql/query
+ :stages
+ [{:lib/type :mbql.stage/mbql
+ :source-table (meta/id :products)
+ :expressions [[:concat
+ {:lib/uuid "e190e161-cecf-4c33-aba7-767d3540b54c", :lib/expression-name "CATEGORY"}
+ [:field
+ {:lib/uuid "20c331ec-949c-4666-ad25-d09148d79614", :base-type :type/Text}
+ (meta/id :products :category)]
+ "2"]]
+ :fields [[:expression
+ {:base-type :type/Text, :lib/uuid "702e8a86-bd62-4e4c-8b38-5cd67c6d41f3"}
+ "CATEGORY"]]}
+ {:lib/type :mbql.stage/mbql
+ :breakout [[:field
+ {:lib/uuid "8d3e962d-1593-411d-99ed-c9fb30bf21c6", :base-type :type/Text}
+ "CATEGORY"]]
+ :aggregation [[:count {:lib/uuid "049bc679-bede-4cae-a28b-0a2b2a18ffd6"}]]
+ :order-by [[:asc
+ {:lib/uuid "4d859243-380d-4cef-9841-8edd0d90becc"}
+ [:field
+ {:lib/uuid "4e1391f8-3d4c-4c7c-a4cc-723cf15eb18a", :base-type :type/Text}
+ "CATEGORY"]]]}]
+ :database (meta/id)})
+ expected (lib.field.resolution/resolve-field-ref
+ query -1
+ [:field {:lib/uuid "00000000-0000-0000-0000-000000000000" :base-type :type/Number} "CATEGORY"])]
+ (is (= expected
+ (lib.field.resolution/resolve-field-ref
+ query -1
+ [:field {:lib/uuid "00000000-0000-0000-0000-000000000000", :base-type :type/Number} "CATEGORY_2"])))
+ (testing "Should be able to resolve nonexistent CATEGORY_3 to existent CATEGORY (i.e., try recursively)"
+ (is (= expected
+ (lib.field.resolution/resolve-field-ref
+ query -1
+ [:field {:lib/uuid "00000000-0000-0000-0000-000000000000", :base-type :type/Number} "CATEGORY_3"])))))))
+
+(deftest ^:parallel fallback-resolve-deduplicated-column-name-to-undeduplicated-name-test-2
+ (testing "Should be able to resolve nonexistent `CATEGORY_3` to existent `CATEGORY_2`"
+ (let [query (lib/query
+ meta/metadata-provider
+ {:lib/type :mbql/query
+ :stages
+ [{:lib/type :mbql.stage/mbql
+ :source-table (meta/id :products)
+ :expressions [[:concat
+ {:lib/uuid "e190e161-cecf-4c33-aba7-767d3540b54c", :lib/expression-name "CATEGORY"}
+ [:field
+ {:lib/uuid "20c331ec-949c-4666-ad25-d09148d79614", :base-type :type/Text}
+ (meta/id :products :category)]
+ "2"]]
+ :fields [[:field
+ {:base-type :type/Text, :lib/uuid "5a26b338-dbf2-4586-82f5-08d1eafde373"}
+ "CATEGORY"]
+ [:expression
+ {:base-type :type/Text, :lib/uuid "702e8a86-bd62-4e4c-8b38-5cd67c6d41f3"}
+ "CATEGORY"]]}
+ {:lib/type :mbql.stage/mbql
+ :breakout [[:field
+ {:lib/uuid "8d3e962d-1593-411d-99ed-c9fb30bf21c6", :base-type :type/Text}
+ "CATEGORY_2"]]
+ :aggregation [[:count {:lib/uuid "049bc679-bede-4cae-a28b-0a2b2a18ffd6"}]]
+ :order-by [[:asc
+ {:lib/uuid "4d859243-380d-4cef-9841-8edd0d90becc"}
+ [:field
+ {:lib/uuid "4e1391f8-3d4c-4c7c-a4cc-723cf15eb18a", :base-type :type/Text}
+ "CATEGORY_2"]]]}]
+ :database (meta/id)})]
+ (is (=? {:lib/deduplicated-name "CATEGORY_2"
+ :lib/original-expression-name "CATEGORY"
+ :lib/original-name "CATEGORY"
+ :lib/source-column-alias "CATEGORY_2"
+ ::lib.field.resolution/fallback-metadata? (symbol "nil #_\"key is not present.\"")}
+ (lib.field.resolution/resolve-field-ref
+ query -1
+ [:field {:lib/uuid "00000000-0000-0000-0000-000000000000", :base-type :type/Number} "CATEGORY_3"]))))))
diff --git a/test/metabase/lib/test_util/places_cam_likes_metadata_provider.cljc b/test/metabase/lib/test_util/places_cam_likes_metadata_provider.cljc
new file mode 100644
index 00000000000..11f2fcbf2a6
--- /dev/null
+++ b/test/metabase/lib/test_util/places_cam_likes_metadata_provider.cljc
@@ -0,0 +1,40 @@
+(ns metabase.lib.test-util.places-cam-likes-metadata-provider
+ (:require
+ [metabase.lib.test-metadata :as meta]
+ [metabase.lib.test-util :as lib.tu]))
+
+(ns metabase.lib.test-util.places-cam-likes-metadata-provider)
+
+(defn id
+ ([]
+ 1)
+ ([table-name]
+ (case table-name
+ :places 1))
+ ([table-name field-name]
+ (case table-name
+ :places (case field-name
+ :id 1
+ :liked 2))))
+
+(def metadata-provider
+ "Mock metadata provider with metadata similar to the `places-cam-likes` test dataset."
+ (lib.tu/mock-metadata-provider
+ {:database (merge meta/database
+ {:id 1})
+ :tables [(merge (meta/table-metadata :venues)
+ {:id 1
+ :db-id 1
+ :name "PLACES"})]
+ ;; TODO -- this metadata is a little incomplete, we can add better fingerprints and what not if we need it in
+ ;; future tests.
+ :fields [(merge (meta/field-metadata :venues :id)
+ {:table-id 1
+ :id 1})
+ (merge (meta/field-metadata :venues :id)
+ {:table-id 1
+ :id 2
+ :name "LIKED"
+ :base-type :type/Boolean
+ :effective-type :type/Boolean
+ :fingerprint nil})]}))
diff --git a/test/metabase/query_processor/util/nest_query_test.clj b/test/metabase/query_processor/util/nest_query_test.clj
index b6790b57c06..71afea782c7 100644
--- a/test/metabase/query_processor/util/nest_query_test.clj
+++ b/test/metabase/query_processor/util/nest_query_test.clj
@@ -1,276 +1,155 @@
(ns metabase.query-processor.util.nest-query-test
(:require
- [clojure.set :as set]
[clojure.test :refer :all]
- [clojure.walk :as walk]
[metabase.driver :as driver]
[metabase.lib.core :as lib]
- [metabase.lib.metadata :as lib.metadata]
[metabase.lib.test-metadata :as meta]
[metabase.lib.test-util :as lib.tu]
[metabase.lib.test-util.macros :as lib.tu.macros]
+ [metabase.lib.test-util.places-cam-likes-metadata-provider :as lib.tu.places-cam-likes-metadata-provider]
[metabase.query-processor :as qp]
[metabase.query-processor.preprocess :as qp.preprocess]
- ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store]
[metabase.query-processor.util.add-alias-info :as add]
[metabase.query-processor.util.nest-query :as nest-query]
- [metabase.test :as mt]
- [metabase.util :as u]))
+ [metabase.test :as mt]))
-;;; TODO (Cam 7/18/25) -- update all the tests that use `with-temp` to use mock metadata providers instead.
-
-;; TODO -- this is duplicated with [[metabase.query-processor.util.add-alias-info-test/remove-source-metadata]]
-(defn- remove-source-metadata [x]
- (walk/postwalk
- (fn [x]
- (if ((every-pred map? :source-metadata) x)
- (dissoc x :source-metadata)
- x))
- x))
-
-(defn- nest-expressions [query]
+(defn- nest-expressions-mbql5 [query]
(driver/with-driver (or driver/*driver* :h2)
(-> query
qp.preprocess/preprocess
- lib/->legacy-MBQL
- :query
- nest-query/nest-expressions
- remove-source-metadata)))
+ nest-query/nest-expressions)))
+
+(defn- nest-expressions [query]
+ (-> query
+ nest-expressions-mbql5
+ lib/->legacy-MBQL
+ :query))
(deftest ^:parallel nest-expressions-test
- (driver/with-driver :h2
- (qp.store/with-metadata-provider meta/metadata-provider
- (is (=? (lib.tu.macros/$ids venues
- {:source-query {:source-table $$venues
- :expressions {"double_price" [:* [:field %price {::add/source-table $$venues
- ::add/source-alias "PRICE"
- ::add/desired-alias "PRICE"}]
- 2]}
- :fields [[:field %price {::add/source-table $$venues
- ::add/source-alias "PRICE"
- ::add/desired-alias "PRICE"}]
- [:expression "double_price" {::add/desired-alias "double_price"}]]}
- :breakout [[:field %price {::add/source-table ::add/source
- ::add/source-alias "PRICE"
- ::add/desired-alias "PRICE"}]
- [:field "double_price" {:base-type :type/Integer
- ::add/source-table ::add/source
- ::add/source-alias "double_price"
- ::add/desired-alias "double_price"}]]
- :aggregation [[:aggregation-options [:count] {::add/desired-alias "count"}]]
- :order-by [[:asc [:field %price {::add/source-table ::add/source
- ::add/source-alias "PRICE"
- ::add/desired-alias "PRICE"}]]
- [:asc [:field "double_price" {::add/source-table ::add/source
- ::add/source-alias "double_price"
- ::add/desired-alias "double_price"}]]]})
- (-> (lib.tu.macros/mbql-query venues
- {:expressions {"double_price" [:* $price 2]}
- :breakout [$price
- [:expression "double_price"]]
- :aggregation [[:count]]})
- qp.preprocess/preprocess
- add/add-alias-info
- nest-expressions))))))
+ (is (=? (lib.tu.macros/$ids venues
+ {:source-query {:source-table $$venues
+ :expressions {"double_price" [:* [:field %price nil]
+ 2]}
+ :fields [[:field %price nil]
+ [:expression "double_price" {}]]}
+ :breakout [[:field "PRICE" {}]
+ [:field "double_price" {}]]
+ :aggregation [[:count]]
+ :order-by [[:asc [:field "PRICE" {}]]
+ [:asc [:field "double_price" {}]]]})
+ (-> (lib/query
+ meta/metadata-provider
+ (lib.tu.macros/mbql-query venues
+ {:expressions {"double_price" [:* $price 2]}
+ :breakout [$price
+ [:expression "double_price"]]
+ :aggregation [[:count]]}))
+ qp.preprocess/preprocess
+ nest-expressions))))
(deftest ^:parallel nest-order-by-expressions-test
(testing "Expressions in an order-by clause result in nesting"
- (driver/with-driver :h2
- (qp.store/with-metadata-provider meta/metadata-provider
- (is (partial= (lib.tu.macros/$ids venues
- {:source-query {:source-table $$venues
- :expressions {"double_price" [:* [:field %price {::add/source-table $$venues
- ::add/source-alias "PRICE"
- ::add/desired-alias "PRICE"}]
- 2]}
- :fields [[:field %price {::add/source-table $$venues
- ::add/source-alias "PRICE"
- ::add/desired-alias "PRICE"}]
- [:expression "double_price" {::add/desired-alias "double_price"}]]}
- :order-by [[:asc *double_price/Integer]]})
- (-> (lib.tu.macros/mbql-query venues
- {:expressions {"double_price" [:* $price 2]}
- :fields [$price
- [:expression "double_price"]]
- :order-by [[:asc [:expression "double_price"]]]})
- qp.preprocess/preprocess
- add/add-alias-info
- nest-expressions)))))))
+ (is (=? (lib.tu.macros/$ids venues
+ {:source-query {:source-table $$venues
+ :expressions {"double_price" [:* [:field %price nil]
+ 2]}
+ :fields [[:field %price nil]
+ [:expression "double_price" {}]]}
+ :order-by [[:asc *double_price/Integer]]})
+ (-> (lib/query
+ meta/metadata-provider
+ (lib.tu.macros/mbql-query venues
+ {:expressions {"double_price" [:* $price 2]}
+ :fields [$price
+ [:expression "double_price"]]
+ :order-by [[:asc [:expression "double_price"]]]}))
+ qp.preprocess/preprocess
+ nest-expressions)))))
(deftest ^:parallel nest-order-by-literal-expressions-test
(testing "Literal expressions in an order-by clause result in nesting"
- (driver/with-driver :h2
- (qp.store/with-metadata-provider meta/metadata-provider
- (is (partial= (lib.tu.macros/$ids venues
- {:source-query {:source-table $$venues
- :expressions {"favorite" [:value "good venue" {:base_type :type/Text}]}
- :fields [[:field %name {::add/source-table $$venues
- ::add/source-alias "NAME"
- ::add/desired-alias "NAME"}]
- [:expression "favorite" {::add/desired-alias "favorite"}]]}
- :filter [:= [:field %name {::add/source-table ::add/source
- ::add/source-alias "NAME"
- ::add/desired-alias "NAME"
- :qp/ignore-coercion true}]
- [:field "favorite" {:base-type :type/Text
- ::add/source-table ::add/source
- ::add/source-alias "favorite"
- ::add/desired-alias "favorite"}]]
- :order-by [[:asc *favorite/Text]]})
- (-> (lib.tu.macros/mbql-query venues
- {:expressions {"favorite" [:value "good venue" {:base_type :type/Text}]}
- :fields [$name
- [:expression "favorite"]]
- :filter [:= $name [:expression "favorite"]]
- :order-by [[:asc [:expression "favorite"]]]})
- qp.preprocess/preprocess
- add/add-alias-info
- nest-expressions)))))))
+ (is (=? (lib.tu.macros/$ids venues
+ {:source-query {:source-table $$venues
+ :expressions {"favorite" [:value "good venue" {:base_type :type/Text}]}
+ :fields [[:field %name nil]
+ [:expression "favorite" {}]]
+ :filter [:=
+ [:field %name nil]
+ [:expression "favorite" {:base-type :type/Text}]]}
+ :order-by [[:asc *favorite/Text]]})
+ (-> (lib/query
+ meta/metadata-provider
+ (lib.tu.macros/mbql-query venues
+ {:expressions {"favorite" [:value "good venue" {:base_type :type/Text}]}
+ :fields [$name
+ [:expression "favorite"]]
+ :filter [:= $name [:expression "favorite"]]
+ :order-by [[:asc [:expression "favorite"]]]}))
+ qp.preprocess/preprocess
+ nest-expressions)))))
(deftest ^:parallel multiple-expressions-test
(testing "Make sure the nested version of the query doesn't mix up expressions if we have ones that reference others"
- (driver/with-driver :h2
- (qp.store/with-metadata-provider meta/metadata-provider
- (is (partial= (lib.tu.macros/$ids venues
- {:source-query {:source-table $$venues
- :expressions {"big_price"
- [:+
- [:field %price {::add/source-table $$venues
- ::add/source-alias "PRICE"
- ::add/desired-alias "PRICE"}]
- 2]
+ (is (=? (lib.tu.macros/$ids venues
+ {:source-query {:source-table $$venues
+ :expressions {"big_price"
+ [:+
+ [:field %price nil]
+ 2]
- "my_cool_new_field"
- [:/
- [:field %price {::add/source-table $$venues
- ::add/source-alias "PRICE"
- ::add/desired-alias "PRICE"}]
- [:expression "big_price" {::add/desired-alias "big_price"}]]}
- :fields [[:field %id {::add/source-table $$venues
- ::add/source-alias "ID"
- ::add/desired-alias "ID"}]
- [:field %price {::add/source-table $$venues
- ::add/source-alias "PRICE"
- ::add/desired-alias "PRICE"}]
- [:expression "big_price" {::add/desired-alias "big_price"}]
- [:expression "my_cool_new_field" {::add/desired-alias "my_cool_new_field"}]]}
- :breakout [[:field "my_cool_new_field" {:base-type :type/Float
- ::add/source-table ::add/source
- ::add/source-alias "my_cool_new_field"
- ::add/desired-alias "my_cool_new_field"}]]
- :order-by [[:asc [:field %id {::add/source-table ::add/source
- ::add/source-alias "ID"}]]]
- :limit 3})
- (-> (lib.tu.macros/mbql-query venues
- {:expressions {"big_price" [:+ $price 2]
- "my_cool_new_field" [:/ $price [:expression "big_price"]]}
- :breakout [[:expression "my_cool_new_field"]]
- :order-by [[:asc $id]]
- :limit 3})
- add/add-alias-info
- nest-expressions)))))))
+ "my_cool_new_field"
+ [:/
+ [:field %price nil]
+ [:expression "big_price" {}]]}
+ :fields [[:expression "my_cool_new_field" {}]
+ [:field %id nil]]}
+ :breakout [[:field "my_cool_new_field" {:base-type :type/Float}]]
+ ;; uh... not sure ordering by ID is legal here??? Since it doesn't appear in the query results??
+ :order-by [[:asc [:field "ID" {}]]
+ [:asc [:field "my_cool_new_field" {:base-type :type/Float}]]]
+ :limit 3})
+ (-> (lib/query
+ meta/metadata-provider
+ (lib.tu.macros/mbql-query venues
+ {:expressions {"big_price" [:+ $price 2]
+ "my_cool_new_field" [:/ $price [:expression "big_price"]]}
+ :breakout [[:expression "my_cool_new_field"]]
+ :order-by [[:asc $id]]
+ :limit 3}))
+ nest-expressions)))))
(deftest ^:parallel nest-expressions-ignore-source-queries-test
(testing (str "When 'raising' :expression clauses, only raise ones in the current level. Handle duplicate expression "
"names correctly.")
- (driver/with-driver :h2
- (qp.store/with-metadata-provider meta/metadata-provider
- (let [query (lib.tu.macros/mbql-query venues
- {:source-query {:source-table $$venues
- :expressions {"x" [:* $price 2]}
- :breakout [$id [:expression "x"]]}
- :expressions {"x" [:* $price 4]}
- :breakout [$id [:expression "x"]]
- :limit 1})]
- (mt/with-native-query-testing-context query
- (is (partial= (lib.tu.macros/$ids venues
- {:source-query {:source-query {:source-query {:source-table $$venues
- :expressions {"x" [:*
- [:field %price #::add{:source-table $$venues
- :source-alias "PRICE"
- :desired-alias "PRICE"}]
- 2]}
- :fields [[:field %id #::add{:source-table $$venues
- :source-alias "ID"
- :desired-alias "ID"}]
- [:field %price #::add{:source-table $$venues
- :source-alias "PRICE"
- :desired-alias "PRICE"}]
- [:expression "x" #::add{:desired-alias "x"}]]}
- :breakout [[:field %id #::add{:source-table ::add/source
- :source-alias "ID"
- :desired-alias "ID"}]
- [:field "x" {:base-type :type/Integer
- ::add/source-table ::add/source
- ::add/source-alias "x"
- ::add/desired-alias "x"}]]}
- :expressions {"x" [:*
- [:field %price #::add{:source-table ::add/source
- :source-alias "PRICE"}]
- 4]}
- :fields [[:field "ID" #::add{:source-table ::add/source
- :source-alias "ID"
- :desired-alias "ID"}]
- [:field "x" {:base-type :type/Integer
- ::add/source-table ::add/source
- ::add/source-alias "x"
- ::add/desired-alias "x"}]
- [:expression "x" #::add{:desired-alias "x_2"}]]}
- :breakout [[:field %id #::add{:source-table ::add/source
- :source-alias "ID"
- :desired-alias "ID"}]
- [:field "x_2" {:base-type :type/Integer
- ::add/source-table ::add/source
- ::add/source-alias "x_2"
- ::add/desired-alias "x_2"}]]
- :limit 1})
- (-> query add/add-alias-info nest-expressions)))))))))
-
-(deftest ^:parallel nest-expressions-ignore-source-queries-from-joins-test
- (testing "Ignores source-query from joins (#20809)"
- (let [query {:source-table 2
- :expressions {"CC" [:+ 1 1]}
- :fields [[:field 33 {:join-alias "Question 4918"}]
- [:field "count" {:join-alias "Question 4918"}]]
- :joins [{:alias "Question 4918"
- :strategy :left-join
- :fields [[:field 33 {:join-alias "Question 4918"}]
- [:field
- "count"
- {:join-alias "Question 4918"}]]
- :condition [:=
- [:field 5 nil]
- [:field 33 {:join-alias "Question 4918"}]]
- :source-card-id 4918
- :source-query {:source-table 4
- ;; nested query has filter values with join-alias that should not
- ;; be selected
- :filter [:=
- [:field 26 {:join-alias "PRODUCTS__via__PRODUCT_ID"}]
- [:value "Doohickey" {}]]
- :aggregation [[:aggregation-options
- [:count]
- {:name "count"}]]
- :breakout [[:field 33 nil]]
- :limit 2
- :order-by [[:asc
- [:field 33 nil]]]
- ;; nested query has an implicit join with conditions that should
- ;; not be selected
- :joins [{:alias "PRODUCTS__via__PRODUCT_ID"
- :strategy :left-join
- :condition [:=
- [:field 33 nil]
- [:field
- 30
- {:join-alias "PRODUCTS__via__PRODUCT_ID"}]]
- :source-table 1
- :fk-field-id 33}]}
- :source-metadata [{:field_ref [:field 33 nil]}
- {:field_ref [:aggregation 0]}]}]}]
- (is (= [[:field 33 {:join-alias "Question 4918"}]
- [:field "count" {:join-alias "Question 4918"}]]
- (#'nest-query/joined-fields query))))))
+ (let [query (lib/query
+ meta/metadata-provider
+ (lib.tu.macros/mbql-query venues
+ {:source-query {:source-table $$venues
+ :expressions {"x" [:* $price 2]}
+ :breakout [$id [:expression "x"]]}
+ :expressions {"x" [:* $price 4]}
+ :breakout [$id [:expression "x"]]
+ :limit 1}))]
+ (mt/with-native-query-testing-context query
+ (is (=? (lib.tu.macros/$ids venues
+ {:source-query {:source-query {:source-query {:source-table $$venues
+ :expressions {"x" [:*
+ [:field %price nil]
+ 2]}
+ :fields [[:field %id nil]
+ #_[:field %price nil]
+ [:expression "x" {}]]}
+ :breakout [[:field "ID" {}]
+ [:field "x" {:base-type :type/Integer}]]}
+ :expressions {"x" [:*
+ [:field %price nil]
+ 4]}
+ :fields [[:field %id nil]
+ [:expression "x" {}]]}
+ :breakout [[:field "ID" {}]
+ [:field "x" {:base-type :type/Integer}]]
+ :limit 1})
+ (nest-expressions query)))))))
(deftest ^:parallel idempotence-test
(testing "A nested query should return the same set of columns as the original"
@@ -282,18 +161,20 @@
:aggregation [[:count]]
;; filter on an implicit join
:filter [:= $product-id->products.category "Doohickey"]})}]})
- query (lib.tu.macros/mbql-query orders
- {:joins [{:source-table "card__1"
- :alias "Question 1"
- :condition [:=
- $product-id
- [:field
- %reviews.product-id
- {:join-alias "Question 1"}]]
- :fields :all}]
- :expressions {"CC" [:+ 1 1]}
- :limit 2})
- nested (assoc query :query (qp.store/with-metadata-provider mp (nest-expressions query)))
+ query (lib/query
+ mp
+ (lib.tu.macros/mbql-query orders
+ {:joins [{:source-table "card__1"
+ :alias "Question 1"
+ :condition [:=
+ $product-id
+ [:field
+ %reviews.product-id
+ {:join-alias "Question 1"}]]
+ :fields :all}]
+ :expressions {"CC" [:+ 1 1]}
+ :limit 2}))
+ nested (assoc query :query (nest-expressions query))
query* (lib/query mp query)
nested* (lib/query mp nested)]
(is (= (map :lib/desired-column-alias (lib/returned-columns query*))
@@ -308,16 +189,15 @@
reviews
{:breakout [$product_id]
:aggregation [[:count]]
- ;; filter on an implicit join
+ ;; filter on an implicit join
:filter [:= $product_id->products.category "Doohickey"]})}]})]
;; the result returned is not important, just important that the query is valid and completes
- (is (vector?
+ (is (seq
(mt/rows
(qp/process-query
(lib/query
mp
- (mt/mbql-query
- orders
+ (mt/mbql-query orders
{:joins [{:source-table "card__1"
:alias "Question 1"
:condition [:=
@@ -329,520 +209,277 @@
:expressions {"CC" [:+ 1 1]}
:limit 2})))))))))
-#_{:clj-kondo/ignore [:metabase/i-like-making-cams-eyes-bleed-with-horrifically-long-tests]}
(deftest ^:parallel nest-expressions-with-joins-test
- (driver/with-driver :h2
- (qp.store/with-metadata-provider meta/metadata-provider
- (testing "If there are any `:joins`, those need to be nested into the `:source-query` as well."
- (is (partial= (lib.tu.macros/$ids venues
- {:source-query {:source-table $$venues
- :joins [{:strategy :left-join
- :condition [:=
- [:field %category-id {::add/source-table $$venues
- ::add/source-alias "CATEGORY_ID"}]
- [:field %category-id {:join-alias "CategoriesStats"
- ::add/source-table "CategoriesStats"
- ::add/source-alias "CATEGORY_ID"}]]
- :source-query {:source-table $$venues
- :aggregation [[:aggregation-options
- [:max [:field %price {::add/source-table $$venues
- ::add/source-alias "PRICE"}]]
- {:name "MaxPrice"
- ::add/desired-alias "MaxPrice"}]
- [:aggregation-options
- [:avg
- [:field
- %price
- {::add/source-table $$venues
- ::add/source-alias "PRICE"}]]
- {:name "AvgPrice"
- ::add/desired-alias "AvgPrice"}]
- [:aggregation-options
- [:min [:field %price {::add/source-table $$venues
- ::add/source-alias "PRICE"}]]
- {:name "MinPrice"
- ::add/desired-alias "MinPrice"}]]
- :breakout [[:field %category-id {::add/source-table $$venues
- ::add/source-alias "CATEGORY_ID"
- ::add/desired-alias "CATEGORY_ID"}]]
- :order-by [[:asc [:field %category-id {::add/source-table $$venues
- ::add/source-alias "CATEGORY_ID"
- ::add/desired-alias "CATEGORY_ID"}]]]}
- :alias "CategoriesStats"
- :fields [[:field %category-id {:join-alias "CategoriesStats"}]
- [:field "MaxPrice" {:join-alias "CategoriesStats"}]
- [:field "AvgPrice" {:join-alias "CategoriesStats"}]
- [:field "MinPrice" {:join-alias "CategoriesStats"}]]}]
- :expressions {"RelativePrice" [:/
- [:field %price {::add/source-table $$venues
- ::add/source-alias "PRICE"
- ::add/desired-alias "PRICE"}]
- [:field "AvgPrice" {:base-type :type/Integer
- :join-alias "CategoriesStats"
- ::add/source-table "CategoriesStats"
- ::add/source-alias "AvgPrice"
- ::add/desired-alias "CategoriesStats__AvgPrice"}]]}
- :fields [[:field %id {::add/source-table $$venues
- ::add/source-alias "ID"
- ::add/desired-alias "ID"}]
- [:field %name {::add/source-table $$venues
- ::add/source-alias "NAME"
- ::add/desired-alias "NAME"}]
- [:field %category-id {::add/source-table $$venues
- ::add/source-alias "CATEGORY_ID"
- ::add/desired-alias "CATEGORY_ID"}]
- [:field %latitude {::add/source-table $$venues
- ::add/source-alias "LATITUDE"
- ::add/desired-alias "LATITUDE"}]
- [:field %longitude {::add/source-table $$venues
- ::add/source-alias "LONGITUDE"
- ::add/desired-alias "LONGITUDE"}]
- [:field %price {::add/source-table $$venues
- ::add/source-alias "PRICE"
- ::add/desired-alias "PRICE"}]
- [:expression "RelativePrice" {::add/desired-alias "RelativePrice"}]
- [:field %category-id {:join-alias "CategoriesStats"
- ::add/source-table "CategoriesStats"
- ::add/source-alias "CATEGORY_ID"
- ::add/desired-alias "CategoriesStats__CATEGORY_ID"}]
- [:field "MaxPrice" {:base-type :type/Integer
- :join-alias "CategoriesStats"
- ::add/source-table "CategoriesStats"
- ::add/source-alias "MaxPrice"
- ::add/desired-alias "CategoriesStats__MaxPrice"}]
- [:field "AvgPrice" {:base-type :type/Float
- :join-alias "CategoriesStats"
- ::add/source-table "CategoriesStats"
- ::add/source-alias "AvgPrice"
- ::add/desired-alias "CategoriesStats__AvgPrice"}]
- [:field "MinPrice" {:base-type :type/Integer
- :join-alias "CategoriesStats"
- ::add/source-table "CategoriesStats"
- ::add/source-alias "MinPrice"
- ::add/desired-alias "CategoriesStats__MinPrice"}]]}
- :breakout [[:field %id {::add/source-table ::add/source
- ::add/source-alias "ID"
- ::add/desired-alias "ID"}]
- [:field %name {::add/source-table ::add/source
- ::add/source-alias "NAME"
- ::add/desired-alias "NAME"}]
- [:field %category-id {::add/source-table ::add/source
- ::add/source-alias "CATEGORY_ID"
- ::add/desired-alias "CATEGORY_ID"}]
- [:field %latitude {::add/source-table ::add/source
- ::add/source-alias "LATITUDE"
- ::add/desired-alias "LATITUDE"}]
- [:field %longitude {::add/source-table ::add/source
- ::add/source-alias "LONGITUDE"
- ::add/desired-alias "LONGITUDE"}]
- [:field %price {::add/source-table ::add/source
- ::add/source-alias "PRICE"
- ::add/desired-alias "PRICE"}]
- [:field "RelativePrice" {:base-type :type/Float
- ::add/source-table ::add/source
- ::add/source-alias "RelativePrice"
- ::add/desired-alias "RelativePrice"}]
- [:field %category-id {:join-alias "CategoriesStats"
- ::add/source-alias "CategoriesStats__CATEGORY_ID"
- ::add/desired-alias "CategoriesStats__CATEGORY_ID"
- ::add/source-table ::add/source}]
- [:field "MaxPrice" {:base-type :type/Integer
- :join-alias "CategoriesStats"
- ::add/source-alias "CategoriesStats__MaxPrice"
- ::add/desired-alias "CategoriesStats__MaxPrice"
- ::add/source-table ::add/source}]
- [:field "AvgPrice" {:base-type :type/Integer
- :join-alias "CategoriesStats"
- ::add/source-alias "CategoriesStats__AvgPrice"
- ::add/desired-alias "CategoriesStats__AvgPrice"
- ::add/source-table ::add/source}]
- [:field "MinPrice" {:base-type :type/Integer
- :join-alias "CategoriesStats"
- ::add/source-alias "CategoriesStats__MinPrice"
- ::add/desired-alias "CategoriesStats__MinPrice"
- ::add/source-table ::add/source}]]
- :limit 3})
- (-> (lib.tu.macros/mbql-query venues
- {:breakout [$id
- $name
- $category-id
- $latitude
- $longitude
- $price
- [:expression "RelativePrice"]
- &CategoriesStats.category-id
- &CategoriesStats.*MaxPrice/Integer
- &CategoriesStats.*AvgPrice/Integer
- &CategoriesStats.*MinPrice/Integer]
- :expressions {"RelativePrice" [:/ $price &CategoriesStats.*AvgPrice/Integer]}
- :joins [{:strategy :left-join
- :condition [:= $category-id &CategoriesStats.category-id]
- :source-query {:source-table $$venues
- :aggregation [[:aggregation-options [:max $price] {:name "MaxPrice"}]
- [:aggregation-options [:avg $price] {:name "AvgPrice"}]
- [:aggregation-options [:min $price] {:name "MinPrice"}]]
- :breakout [$category-id]}
- :alias "CategoriesStats"
- :fields :all}]
- :limit 3})
- qp.preprocess/preprocess
- add/add-alias-info
- nest-expressions)))))))
+ (testing "If there are any `:joins`, those need to be nested into the `:source-query` as well."
+ (is (=? (lib.tu.macros/$ids venues
+ {:source-query {:source-table $$venues
+ :joins [{:strategy :left-join
+ :condition [:=
+ [:field %category-id nil]
+ [:field %category-id {:join-alias "CategoriesStats"}]]
+ :source-query {:source-table $$venues
+ :aggregation [[:aggregation-options
+ [:max [:field %price nil]]
+ {:name "MaxPrice"}]
+ [:aggregation-options
+ [:avg
+ [:field %price nil]]
+ {:name "AvgPrice"}]
+ [:aggregation-options
+ [:min [:field %price nil]]
+ {:name "MinPrice"}]]
+ :breakout [[:field %category-id nil]]
+ :order-by [[:asc [:field %category-id nil]]]}
+ :alias "CategoriesStats"
+ :fields [[:field %category-id {:join-alias "CategoriesStats"}]
+ [:field "MaxPrice" {:join-alias "CategoriesStats"}]
+ [:field "AvgPrice" {:join-alias "CategoriesStats"}]
+ [:field "MinPrice" {:join-alias "CategoriesStats"}]]}]
+ :expressions {"RelativePrice" [:/
+ [:field %price nil]
+ [:field "AvgPrice" {:join-alias "CategoriesStats"}]]}
+ :fields [[:field %id nil]
+ [:field %name nil]
+ [:field %category-id nil]
+ [:field %latitude nil]
+ [:field %longitude nil]
+ [:field %price nil]
+ [:expression "RelativePrice" {}]
+ [:field %category-id {:join-alias "CategoriesStats"}]
+ [:field "MaxPrice" {:join-alias "CategoriesStats"}]
+ [:field "AvgPrice" {:join-alias "CategoriesStats"}]
+ [:field "MinPrice" {:join-alias "CategoriesStats"}]]}
+ :breakout [[:field "ID" {}]
+ [:field "NAME" {}]
+ [:field "CATEGORY_ID" {}]
+ [:field "LATITUDE" {}]
+ [:field "LONGITUDE" {}]
+ [:field "PRICE" {}]
+ [:field "RelativePrice" {}]
+ [:field "CategoriesStats__CATEGORY_ID" {}]
+ [:field "CategoriesStats__MaxPrice" {}]
+ [:field "CategoriesStats__AvgPrice" {}]
+ [:field "CategoriesStats__MinPrice" {}]]
+ :limit 3})
+ (-> (lib/query
+ meta/metadata-provider
+ (lib.tu.macros/mbql-query venues
+ {:breakout [$id
+ $name
+ $category-id
+ $latitude
+ $longitude
+ $price
+ [:expression "RelativePrice"]
+ &CategoriesStats.category-id
+ &CategoriesStats.*MaxPrice/Integer
+ &CategoriesStats.*AvgPrice/Integer
+ &CategoriesStats.*MinPrice/Integer]
+ :expressions {"RelativePrice" [:/ $price &CategoriesStats.*AvgPrice/Integer]}
+ :joins [{:strategy :left-join
+ :condition [:= $category-id &CategoriesStats.category-id]
+ :source-query {:source-table $$venues
+ :aggregation [[:aggregation-options [:max $price] {:name "MaxPrice"}]
+ [:aggregation-options [:avg $price] {:name "AvgPrice"}]
+ [:aggregation-options [:min $price] {:name "MinPrice"}]]
+ :breakout [$category-id]}
+ :alias "CategoriesStats"
+ :fields :all}]
+ :limit 3}))
+ qp.preprocess/preprocess
+ nest-expressions)))))
(deftest ^:parallel nest-expressions-eliminate-duplicate-coercion-test
(testing "If coercion happens in the source query, don't do it a second time in the parent query (#12430)"
- (driver/with-driver :h2
- (qp.store/with-metadata-provider (lib.tu/merged-mock-metadata-provider
- meta/metadata-provider
- {:fields [{:id (meta/id :venues :price)
- :coercion-strategy :Coercion/UNIXSeconds->DateTime
- :effective-type :type/DateTime}]})
- (is (=? (lib.tu.macros/$ids venues
- {:source-query {:source-table $$venues
- :expressions {"test" [:* 1 1]}
- :fields [[:field %price {::add/source-table $$venues
- ::add/source-alias "PRICE"
- ::add/desired-alias "PRICE"}]
- [:expression "test" {::add/desired-alias "test"}]]}
- :breakout [[:field %price {:temporal-unit :day
- :qp/ignore-coercion true
- ::add/source-table ::add/source
- ::add/source-alias "PRICE"
- ::add/desired-alias "PRICE"}]
- [:field "test" {:base-type :type/Integer
- ::add/source-table ::add/source
- ::add/source-alias "test"
- ::add/desired-alias "test"}]]
- :limit 1})
- (-> (lib.tu.macros/mbql-query venues
- {:expressions {"test" [:* 1 1]}
- :breakout [$price
- [:expression "test"]]
- :limit 1})
- add/add-alias-info
- nest-expressions)))))))
+ (let [mp (lib.tu/merged-mock-metadata-provider
+ meta/metadata-provider
+ {:fields [{:id (meta/id :venues :price)
+ :coercion-strategy :Coercion/UNIXSeconds->DateTime
+ :effective-type :type/DateTime}]})]
+ (is (=? (lib.tu.macros/$ids venues
+ {:source-query {:source-table $$venues
+ :expressions {"test" [:* 1 1]}
+ :fields [[:field %price {:temporal-unit (symbol "nil #_\"key is not present.\"")}]
+ [:expression "test" {}]]}
+ :breakout [[:field "PRICE" {:temporal-unit :day}]
+ [:field "test" {}]]
+ :limit 1})
+ (-> (lib/query
+ mp
+ (lib.tu.macros/mbql-query venues
+ {:expressions {"test" [:* 1 1]}
+ :breakout [$price
+ [:expression "test"]]
+ :limit 1}))
+ nest-expressions))))))
(deftest ^:parallel multiple-joins-with-expressions-test
(testing "We should be able to compile a complicated query with multiple joins and expressions correctly"
- (driver/with-driver :h2
- (mt/dataset test-data
- (qp.store/with-metadata-provider meta/metadata-provider
- (is (=? (:query (merge (let [products-category (lib.tu.macros/$ids orders
- [:field %products.category {:join-alias "PRODUCTS__via__PRODUCT_ID"
- ::add/source-table ::add/source
- ::add/source-alias "PRODUCTS__via__PRODUCT_ID__CATEGORY"
- ::add/desired-alias "PRODUCTS__via__PRODUCT_ID__CATEGORY"}])
- created-at (lib.tu.macros/$ids orders
- [:field %created-at {:temporal-unit :year
- :qp/ignore-coercion true
- ::add/source-table ::add/source
- ::add/source-alias "CREATED_AT"
- ::add/desired-alias "CREATED_AT"}])
- pivot-grouping [:field "pivot-grouping" {:base-type :type/Float
- ::add/source-table ::add/source
- ::add/source-alias "pivot-grouping"
- ::add/desired-alias "pivot-grouping"}]]
- (lib.tu.macros/mbql-query orders
- {:breakout [products-category created-at pivot-grouping]
- :aggregation [[:aggregation-options [:count] {:name "count"
- ::add/desired-alias "count"}]]
- :order-by [[:asc products-category]
- [:asc created-at]
- [:asc pivot-grouping]]}))
- (lib.tu.macros/mbql-query orders
- {:source-query (let [product-id [:field %product-id {::add/source-table $$orders
- ::add/source-alias "PRODUCT_ID"
- ::add/desired-alias "PRODUCT_ID"}]
- created-at [:field %created-at {::add/source-table $$orders
- ::add/source-alias "CREATED_AT"
- ::add/desired-alias "CREATED_AT"}]
- pivot-grouping [:expression "pivot-grouping" {::add/desired-alias "pivot-grouping"}]
- ;; TODO: The order here is not deterministic! It's coming
- ;; from [[metabase.query-processor.util.transformations.nest-breakouts]]
- ;; or [[metabase.query-processor.util.nest-query]], which walks
- ;; the query looking for refs in an arbitrary order, and returns
- ;; `m/distinct-by` over that random order. Changing the map keys
- ;; on the inner query can perturb this order; if you cause this
- ;; test to fail based on shuffling the order of these joined
- ;; fields, just edit the expectation to match the new order.
- ;; Tech debt issue: #39396
- products-id [:field %products.id {:join-alias "PRODUCTS__via__PRODUCT_ID"
- ::add/source-table "PRODUCTS__via__PRODUCT_ID"
- ::add/source-alias "ID"
- ::add/desired-alias "PRODUCTS__via__PRODUCT_ID__ID"}]
- products-category [:field %products.category {:join-alias "PRODUCTS__via__PRODUCT_ID"
- ::add/source-table "PRODUCTS__via__PRODUCT_ID"
- ::add/source-alias "CATEGORY"
- ::add/desired-alias "PRODUCTS__via__PRODUCT_ID__CATEGORY"}]]
- {:source-table $$orders
- :joins [{:source-query {:source-table $$products}
- :alias "PRODUCTS__via__PRODUCT_ID"
- :condition [:=
- [:field %product-id {::add/source-table $$orders
- ::add/source-alias "PRODUCT_ID"}]
- [:field %products.id {:join-alias "PRODUCTS__via__PRODUCT_ID"
- ::add/source-table "PRODUCTS__via__PRODUCT_ID"
- ::add/source-alias "ID"}]]
- :strategy :left-join
- :fk-field-id %product-id}]
- :expressions {"pivot-grouping" [:abs 0]}
- :fields [product-id
- created-at
- pivot-grouping
- products-id
- products-category]})})))
- (-> (lib.tu.macros/mbql-query orders
- {:aggregation [[:aggregation-options [:count] {:name "count"}]]
- :breakout [&PRODUCTS__via__PRODUCT_ID.products.category
- !year.created-at
- [:expression "pivot-grouping"]]
- :expressions {"pivot-grouping" [:abs 0]}
- :order-by [[:asc &PRODUCTS__via__PRODUCT_ID.products.category]
- [:asc !year.created-at]
- [:asc [:expression "pivot-grouping"]]]
- :joins [{:source-table $$products
- :strategy :left-join
- :alias "PRODUCTS__via__PRODUCT_ID"
- :fk-field-id %product-id
- :condition [:= $product-id &PRODUCTS__via__PRODUCT_ID.products.id]}]})
- add/add-alias-info
- nest-expressions
- ;; I'm tired of dealing with the nondeterministic order mentioned above, so just sort them by ID
- ;; and call it a day for now.
- (update-in [:source-query :fields] (fn [fields]
- (concat
- (take 3 fields)
- (sort-by second (drop 3 fields)))))))))))))
+ (is (=? (:query (merge (let [products-category (lib.tu.macros/$ids orders
+ [:field %products.category {:join-alias "PRODUCTS__via__PRODUCT_ID"}])
+ created-at (lib.tu.macros/$ids orders
+ [:field %created-at {:temporal-unit :year
+ :qp/ignore-coercion true}])
+ pivot-grouping [:field "pivot-grouping" {:base-type :type/Float}]]
+ (lib.tu.macros/mbql-query orders
+ {:breakout [products-category
+ created-at
+ pivot-grouping]
+ :aggregation [[:aggregation-options [:count] {:name "count"}]]
+ :order-by [[:asc products-category]
+ [:asc created-at]
+ [:asc pivot-grouping]]}))
+ (lib.tu.macros/mbql-query orders
+ ;; TODO: The order here is not deterministic! It's coming
+ ;; from [[metabase.query-processor.util.transformations.nest-breakouts]]
+ ;; or [[metabase.query-processor.util.nest-query]], which walks the query looking for
+ ;; refs in an arbitrary order, and returns `m/distinct-by` over that random order.
+ ;; Changing the map keys on the inner query can perturb this order; if you cause this
+ ;; test to fail based on shuffling the order of these joined fields, just edit the
+ ;; expectation to match the new order. Tech debt issue: #39396
+ {:source-query {:source-table $$orders
+ :joins [{:source-query {:source-table $$products}
+ :alias "PRODUCTS__via__PRODUCT_ID"
+ :condition [:=
+ [:field %product-id nil]
+ [:field %products.id {:join-alias "PRODUCTS__via__PRODUCT_ID"}]]
+ :strategy :left-join
+ :fk-field-id %product-id}]
+ :expressions {"pivot-grouping" [:abs 0]}
+ :fields [[:field %products.category {:join-alias "PRODUCTS__via__PRODUCT_ID"}]
+ [:field %orders.created-at nil]
+ [:expression "pivot-grouping" {:base-type :type/Integer}]]}})))
+ (-> (lib/query
+ meta/metadata-provider
+ (lib.tu.macros/mbql-query orders
+ {:aggregation [[:aggregation-options [:count] {:name "count"}]]
+ :breakout [&PRODUCTS__via__PRODUCT_ID.products.category
+ !year.created-at
+ [:expression "pivot-grouping"]]
+ :expressions {"pivot-grouping" [:abs 0]}
+ :order-by [[:asc &PRODUCTS__via__PRODUCT_ID.products.category]
+ [:asc !year.created-at]
+ [:asc [:expression "pivot-grouping"]]]
+ :joins [{:source-table $$products
+ :strategy :left-join
+ :alias "PRODUCTS__via__PRODUCT_ID"
+ :fk-field-id %product-id
+ :condition [:= $product-id &PRODUCTS__via__PRODUCT_ID.products.id]}]}))
+ nest-expressions)))))
(deftest ^:parallel uniquify-aliases-test
- (driver/with-driver :h2
- (qp.store/with-metadata-provider meta/metadata-provider
- (is (=? (lib.tu.macros/$ids products
- {:source-query {:source-table $$products
- :expressions {"CATEGORY" [:concat
- [:field %category {::add/source-table $$products
- ::add/source-alias "CATEGORY"
- ::add/desired-alias "CATEGORY"}]
- "2"]}
- :fields [[:field %category {::add/source-table $$products
- ::add/source-alias "CATEGORY"
- ::add/desired-alias "CATEGORY"}]
- [:expression "CATEGORY" {::add/desired-alias "CATEGORY_2"}]]}
- :breakout [[:field "CATEGORY_2" {:base-type :type/Text
- ::add/source-table ::add/source
- ::add/source-alias "CATEGORY_2"
- ::add/desired-alias "CATEGORY_2"}]]
- :aggregation [[:aggregation-options [:count] {::add/desired-alias "count"}]]
- :order-by [[:asc [:field "CATEGORY_2" {:base-type :type/Text
- ::add/source-table ::add/source
- ::add/source-alias "CATEGORY_2"
- ::add/desired-alias "CATEGORY_2"}]]]
- :limit 1})
- (-> (lib.tu.macros/mbql-query products
- {:expressions {"CATEGORY" [:concat $category "2"]}
- :breakout [:expression "CATEGORY"]
- :aggregation [[:count]]
- :order-by [[:asc [:expression "CATEGORY"]]]
- :limit 1})
- qp.preprocess/preprocess
- add/add-alias-info
- lib/->legacy-MBQL
- :query
- nest-query/nest-expressions))))))
+ (is (=? (lib.tu.macros/$ids products
+ {:source-query {:source-table $$products
+ :expressions {"CATEGORY" [:concat
+ [:field %category nil]
+ "2"]}
+ :fields [[:expression "CATEGORY" {}]]}
+ :breakout [[:field "CATEGORY" {:base-type :type/Text}]]
+ :aggregation [[:count]]
+ :order-by [[:asc [:field "CATEGORY" {:base-type :type/Text}]]]
+ :limit 1})
+ (-> (lib/query
+ meta/metadata-provider
+ (lib.tu.macros/mbql-query products
+ {:expressions {"CATEGORY" [:concat $category "2"]}
+ :breakout [:expression "CATEGORY"]
+ :aggregation [[:count]]
+ :order-by [[:asc [:expression "CATEGORY"]]]
+ :limit 1}))
+ nest-expressions))))
(deftest ^:parallel uniquify-aliases-test-2
- (driver/with-driver :h2
- (mt/dataset test-data
- (qp.store/with-metadata-provider meta/metadata-provider
- (testing "multi-stage query with an expression name that matches a table column (#39059)"
- (is (=? (lib.tu.macros/mbql-query orders
- {:source-query {:breakout [[:field %id {}]
- [:field %subtotal {}]
- ;; Then exported as DISCOUNT from the middle layer.
- [:field "DISCOUNT_2" {:base-type :type/Float
- ::add/source-alias "DISCOUNT_2"
- ::add/desired-alias "DISCOUNT_2"}]]
- :source-query {:expressions {"DISCOUNT" [:coalesce [:field %discount {}] 0]}
- :fields [[:field %id {::add/desired-alias "ID"}]
- [:field %subtotal {::add/desired-alias "SUBTOTAL"}]
- [:field %discount {::add/desired-alias "DISCOUNT"}]
- ;; Exported as DISCOUNT_2 from this inner query.
- [:expression "DISCOUNT"
- {::add/desired-alias "DISCOUNT_2"}]]
- :source-table $$orders}}
- :source-query/model? true
- :breakout [[:field "ID" {}]
- [:field "SUBTOTAL" {}]
- [:field "DISCOUNT" {:base-type :type/Float
- ::add/source-alias "DISCOUNT"
- ::add/desired-alias "DISCOUNT"}]]})
- (-> (lib.tu.macros/mbql-query orders
- {:source-query {:expressions {"DISCOUNT" [:coalesce $discount 0]}
- :breakout [$id
- $subtotal
- [:expression "DISCOUNT"]]
- :source-table $$orders}
- :source-query/model? true
- :breakout [[:field "ID" {:base-type :type/Integer}]
- [:field "SUBTOTAL" {:base-type :type/Float}]
- [:field "DISCOUNT" {:base-type :type/Float}]]})
- qp.preprocess/preprocess
- add/add-alias-info
- lib/->legacy-MBQL
- :query
- nest-query/nest-expressions
- (->> (assoc {:database (meta/id)
- :type :query}
- :query))))))))))
-
-(defn- readable-query
- "Attempt to make the results of [[add/add-alias-info]] and [[nest-query/nest-expressions]] a little less noisy so
- they're actually readable/debuggable."
- [query]
- (letfn [(inner-query? [form]
- (and (map? form)
- ((some-fn :source-query :source-table) form)))
- (inner-query [form]
- (select-keys form [:source-query
- :expressions
- :breakout
- :aggregation
- :fields]))
- (ref-options-map? [form]
- (and (map? form)
- (::add/desired-alias form)))
- (table-symbol [table-id]
- (let [table-name (:name (lib.metadata/table
- (qp.store/metadata-provider)
- table-id))]
- (symbol (str "$$" (u/lower-case-en table-name)))))
- (ref-options-map [form]
- (-> (select-keys form [::add/source-table ::add/source-alias ::add/desired-alias :temporal-unit :bucketing :join-alias])
- (set/rename-keys {::add/source-table :table, ::add/source-alias :source, ::add/desired-alias :desired})
- (update :table (fn [table]
- (if (integer? table)
- (table-symbol table)
- (if (= table ::add/source)
- :source
- table))))))
- (field-ref? [form]
- (and (vector? form)
- (= (first form) :field)))
- (field-symbol [field-id]
- (let [field-name (:name (lib.metadata/field (qp.store/metadata-provider) field-id))]
- (symbol (str \% (u/->kebab-case-en field-name)))))
- (field-ref [form]
- (let [[_tag id-or-name opts] form]
- [:field
- (cond-> id-or-name (pos-int? id-or-name) field-symbol)
- opts]))]
- (walk/postwalk
- (fn [form]
- (cond-> form
- (inner-query? form) inner-query
- (field-ref? form) field-ref
- (ref-options-map? form) ref-options-map))
- query)))
+ (testing "multi-stage query with an expression name that matches a table column (#39059)"
+ (is (=? (-> (lib.tu.macros/mbql-query orders
+ {:source-query {:breakout [[:field "ID" {}]
+ [:field "SUBTOTAL" {}]
+ [:field "DISCOUNT" {}]]
+ :source-query {:expressions {"DISCOUNT" [:coalesce [:field %discount nil] 0]}
+ :fields [[:field %id nil]
+ [:field %subtotal nil]
+ [:expression "DISCOUNT" {}]]
+ :source-table $$orders}}
+ :breakout [[:field "ID" {}]
+ [:field "SUBTOTAL" {}]
+ [:field "DISCOUNT" {:base-type :type/Float}]]})
+ :query)
+ (-> (lib/query
+ meta/metadata-provider
+ (lib.tu.macros/mbql-query orders
+ {:source-query {:expressions {"DISCOUNT" [:coalesce $discount 0]}
+ :breakout [$id
+ $subtotal
+ [:expression "DISCOUNT"]]
+ :source-table $$orders}
+ :source-query/model? true
+ :breakout [[:field "ID" {:base-type :type/Integer}]
+ [:field "SUBTOTAL" {:base-type :type/Float}]
+ [:field "DISCOUNT" {:base-type :type/Float}]]}))
+ nest-expressions)))))
(deftest ^:parallel do-not-remove-fields-when-referred-to-with-nominal-refs-test
(testing "Don't remove fields if they are used in the next stage with a nominal field literal ref"
- (qp.store/with-metadata-provider meta/metadata-provider
- (driver/with-driver :h2
- (let [query (lib.tu.macros/$ids products
- {:source-query {:source-table $$products
- :fields [[:field %id nil]
- [:field %ean nil]
- [:field %title nil]
- [:field %category nil]
- [:field %vendor nil]
- [:field %price nil]
- [:field %rating nil]
- [:field %created-at {:temporal-unit :default}]]}
- :expressions {"pivot-grouping" [:abs 0]}
- :breakout [[:field "CATEGORY" {:base-type :type/Text}]
- [:field "CREATED_AT" {:base-type :type/DateTime, :temporal-unit :month}]
- [:expression "pivot-grouping"]]
- :aggregation [[:aggregation-options [:count] {:name "count"}]]})
- query' (add/add-alias-info query)]
- (testing (str "with alias info:\n" (u/pprint-to-str (readable-query query')))
- (is (=? '{:source-query {:source-query {:fields [[:field %id {}]
- [:field %ean {}]
- [:field %title {}]
- [:field %category {}]
- [:field %vendor {}]
- [:field %price {}]
- [:field %rating {}]
- [:field %created-at {}]]}
- :expressions {"pivot-grouping" [:abs 0]}
- :fields [[:field "CATEGORY" {}]
- [:field "CREATED_AT" {}]
- [:expression "pivot-grouping" {}]]}
- :breakout [[:field "CATEGORY" {}]
- [:field "CREATED_AT" {}]
- [:field "pivot-grouping" {}]]
- :aggregation [[:aggregation-options [:count] {}]]}
- (-> query'
- nest-query/nest-expressions
- readable-query)))))))))
+ (let [query (lib/query
+ meta/metadata-provider
+ (lib.tu.macros/mbql-query products
+ {:source-query {:source-table $$products
+ :fields [[:field %id nil]
+ [:field %ean nil]
+ [:field %title nil]
+ [:field %category nil]
+ [:field %vendor nil]
+ [:field %price nil]
+ [:field %rating nil]
+ [:field %created-at {:temporal-unit :default}]]}
+ :expressions {"pivot-grouping" [:abs 0]}
+ :breakout [[:field "CATEGORY" {:base-type :type/Text}]
+ [:field "CREATED_AT" {:base-type :type/DateTime, :temporal-unit :month}]
+ [:expression "pivot-grouping"]]
+ :aggregation [[:aggregation-options [:count] {:name "count"}]]}))]
+ (is (=? (lib.tu.macros/$ids products
+ {:source-query {:source-query {:fields [[:field %id nil]
+ [:field %ean nil]
+ [:field %title nil]
+ [:field %category nil]
+ [:field %vendor nil]
+ [:field %price nil]
+ [:field %rating nil]
+ [:field %created-at {:temporal-unit :default}]]}
+ :expressions {"pivot-grouping" [:abs 0]}
+ :fields [[:field "CATEGORY" {}]
+ [:field "CREATED_AT" {}]
+ [:expression "pivot-grouping" {}]]}
+ :breakout [[:field "CATEGORY" {}]
+ [:field "CREATED_AT" {}]
+ [:field "pivot-grouping" {}]]
+ :aggregation [[:aggregation-options [:count] {}]]})
+ (nest-expressions query))))))
(deftest ^:parallel nest-expressions-ignores-temporal-units-from-joined-fields
(testing "clear temporal units from joined fields #48058"
- (driver/with-driver :h2
- (qp.store/with-metadata-provider meta/metadata-provider
- (is (=? {:source-query
- {:fields
- [[:field
- (meta/id :orders :user-id)
- {::add/source-table (meta/id :orders)
- ::add/source-alias "USER_ID"
- ::add/desired-alias "USER_ID"}]
- [:field
- (meta/id :orders :total)
- {::add/source-table (meta/id :orders)
- ::add/source-alias "TOTAL"
- ::add/desired-alias "TOTAL"}]
- [:expression
- "double_total"
- {::add/desired-alias "double_total"}]
- ;; TODO: The order here is not deterministic! It's coming
- ;; from [[metabase.query-processor.util.transformations.nest-breakouts]]
- ;; or [[metabase.query-processor.util.nest-query]], which walks the query looking for refs in an
- ;; arbitrary order, and returns `m/distinct-by` over that random order. Changing the map keys on the
- ;; inner query can perturb this order; if you cause this test to fail based on shuffling the order of
- ;; these joined fields, just edit the expectation to match the new order. Tech debt issue: #39396
- [:field
- (meta/id :people :id)
- {:join-alias "p"
- ::add/source-table "p"
- ::add/source-alias "ID"
- ::add/desired-alias "p__ID"}]
- [:field
- (meta/id :people :created-at)
- {:temporal-unit (symbol "nil #_\"key is not present.\"")
- ::add/source-alias "CREATED_AT"
- :join-alias "p"
- ::add/desired-alias "p__CREATED_AT"
- ::add/source-table "p"}]]}}
- (-> (lib.tu.macros/mbql-query orders
- {:expressions {"double_total" [:* $total 2]}
- ;; this is a broken field ref! It should use the join alias `p`. Luckily
- ;; the [[metabase.query-processor.middleware.fix-bad-field-id-refs]] middleware should fix it
- ;; for us.
- :breakout [!hour-of-day.people.created-at
- [:expression "double_total"]]
- :aggregation [[:count]]
- :joins [{:source-table $$people
- :alias "p"
- :condition [:= $user-id &p.people.id]}]})
- qp.preprocess/preprocess
- add/add-alias-info
- nest-expressions
- ;; I'm tired of dealing with the nondeterministic order mentioned above, so just sort them by ID and
- ;; call it a day for now.
- (update-in [:source-query :fields] (fn [fields]
- (concat
- (take 3 fields)
- (sort-by second (drop 3 fields))))))))))))
+ ;; TODO: The order here is not deterministic! It's coming
+ ;; from [[metabase.query-processor.util.transformations.nest-breakouts]]
+ ;; or [[metabase.query-processor.util.nest-query]], which walks the query looking for refs in an arbitrary order
+ ;; and returns `m/distinct-by` over that random order. Changing the map keys on the inner query can perturb this
+ ;; order; if you cause this test to fail based on shuffling the order of these joined fields, just edit the
+ ;; expectation to match the new order. Tech debt issue: #39396
+ (is (=? {:source-query {:fields [[:field (meta/id :people :created-at) {:temporal-unit (symbol "nil #_\"key is not present.\"")
+ :join-alias "p"}]
+ [:expression "double_total" {}]]}}
+ (-> (lib/query
+ meta/metadata-provider
+ (lib.tu.macros/mbql-query orders
+ {:expressions {"double_total" [:* $total 2]}
+ ;; this is a broken field ref! It should use the join alias `p`. Luckily
+ ;; the [[metabase.query-processor.middleware.fix-bad-field-id-refs]] middleware should fix it
+ ;; for us.
+ :breakout [!hour-of-day.people.created-at
+ [:expression "double_total"]]
+ :aggregation [[:count]]
+ :joins [{:source-table $$people
+ :alias "p"
+ :condition [:= $user-id &p.people.id]}]}))
+ nest-expressions)))))
(deftest ^:parallel wonky-breakout-test
(let [mp (lib.tu/mock-metadata-provider
@@ -866,63 +503,96 @@
{:join-alias "model A - Product"}]]}]})}]})
query (lib/query
mp
- {:database (meta/id)
- :stages [{:lib/type :mbql.stage/mbql
- :source-card 2
- :expressions [[:abs {:lib/expression-name "pivot-grouping"} 0]]
- :aggregation [[:sum {} [:field {:base-type :type/Number} "SUBTOTAL"]]]
- :breakout [[:field {:base-type :type/Number, :join-alias "model A - Product"}
- "Rating Bucket"]
- [:expression {:base-type :type/Integer, :effective-type :type/Integer}
- "pivot-grouping"]]}]
- :lib/type :mbql/query})]
- (qp.store/with-metadata-provider mp
- (driver/with-driver :h2
- (let [stages (-> query
- qp.preprocess/preprocess
- add/add-alias-info
- lib/->legacy-MBQL
- :query
- nest-query/nest-expressions
- (->> (lib/query-from-legacy-inner-query mp (meta/id)))
- :stages
- (->> (map (fn [stage]
- (into []
- (comp (mapcat stage)
- (map lib/options)
- (map ::add/desired-alias))
- [:breakout :aggregation :fields])))))]
- (is (= 3
- (count stages)))
- (testing "first stage (Card 1)"
- (is (= ["ID"
- "USER_ID"
- "PRODUCT_ID"
- "SUBTOTAL"
- "TAX"
- "TOTAL"
- "DISCOUNT"
- "CREATED_AT"
- "QUANTITY"
- "model A - Product__ID"
- "model A - Product__EAN"
- "model A - Product__TITLE"
- "model A - Product__CATEGORY"
- "model A - Product__VENDOR"
- "model A - Product__PRICE"
- "model A - Product__RATING"
- "model A - Product__CREATED_AT"
- "model A - Product__Rating Bucket"]
- (nth stages 0))))
- (testing "second stage (Card 2)"
- (is (= ["ID"
- "SUBTOTAL"
- "CREATED_AT"
- "model A - Product__Rating Bucket"
- "pivot-grouping"]
- (nth stages 1))))
- (testing "third stage (original query)"
- (is (= ["model A - Product__Rating Bucket"
- "pivot-grouping"
- "sum"]
- (nth stages 2)))))))))
+ (lib/query
+ mp
+ {:database (meta/id)
+ :stages [{:lib/type :mbql.stage/mbql
+ :source-card 2
+ :expressions [[:abs {:lib/expression-name "pivot-grouping"} 0]]
+ :aggregation [[:sum {} [:field {:base-type :type/Number} "SUBTOTAL"]]]
+ :breakout [[:field {:base-type :type/Number, :join-alias "model A - Product"}
+ "Rating Bucket"]
+ [:expression {:base-type :type/Integer, :effective-type :type/Integer}
+ "pivot-grouping"]]}]
+ :lib/type :mbql/query}))
+ stages (-> query
+ nest-expressions-mbql5
+ (as-> $query (driver/with-driver :h2
+ (add/add-alias-info $query)))
+ :stages
+ (->> (map (fn [stage]
+ (into []
+ (comp (mapcat stage)
+ (map lib/options)
+ (map ::add/desired-alias))
+ [:breakout :aggregation :fields])))))]
+ (is (= 3
+ (count stages)))
+ (testing "first stage (Card 1)"
+ (is (= ["ID"
+ "USER_ID"
+ "PRODUCT_ID"
+ "SUBTOTAL"
+ "TAX"
+ "TOTAL"
+ "DISCOUNT"
+ "CREATED_AT"
+ "QUANTITY"
+ "model A - Product__ID"
+ "model A - Product__EAN"
+ "model A - Product__TITLE"
+ "model A - Product__CATEGORY"
+ "model A - Product__VENDOR"
+ "model A - Product__PRICE"
+ "model A - Product__RATING"
+ "model A - Product__CREATED_AT"
+ "model A - Product__Rating Bucket"]
+ (nth stages 0))))
+ (testing "second stage (Card 2)"
+ (is (= ["SUBTOTAL"
+ "model A - Product__Rating Bucket"
+ "pivot-grouping"]
+ (nth stages 1))))
+ (testing "third stage (original query)"
+ (is (= ["model A - Product__Rating Bucket"
+ "pivot-grouping"
+ "sum"]
+ (nth stages 2))))))
+
+(deftest ^:parallel literal-boolean-expressions-and-fields-in-conditions-test
+ (testing "mixing Field ID refs and Field Name refs to the same column should not result in broken queries"
+ (let [true-value [:value true {:base_type :type/Boolean}]
+ false-value [:value false {:base_type :type/Boolean}]
+ mp lib.tu.places-cam-likes-metadata-provider/metadata-provider
+ query (lib/query
+ mp
+ {:database 1
+ :type :query
+ :query {:expressions {"T" true-value, "F" false-value}
+ :source-query {:source-table 1
+ :fields [[:field 2 nil]]}
+ :aggregation [[:count-where [:expression "T"]]
+ [:count-where [:expression "F"]]
+ ;; only a true psycho would do this
+ [:count-where [:field "LIKED" {:base-type :type/Boolean}]]
+ [:count-where [:field 2 nil]]]
+ :filter [:or
+ [:field 2 nil]
+ [:field "LIKED" {:base-type :type/Boolean}]
+ [:expression "T"]]}})]
+ (is (= {:aggregation [[:count-where [:field "T" {:base-type :type/Boolean}]]
+ [:count-where [:field "F" {:base-type :type/Boolean}]]
+ [:count-where [:field "LIKED" {:base-type :type/Boolean}]]
+ [:count-where [:field "LIKED" {:base-type :type/Boolean}]]]
+ :source-query {:source-query {:fields [[:field 2 nil]], :source-table 1}
+ :expressions {"F" [:value false {:base_type :type/Boolean}]
+ "T" [:value true {:base_type :type/Boolean}]}
+ :fields [[:expression "T" {:base-type :type/Boolean}]
+ [:expression "F" {:base-type :type/Boolean}]
+ ;; make sure we do not include duplicate entries for `LIKED`/Field 2
+ [:field "LIKED" {:base-type :type/Boolean}]]
+ :filter [:or
+ [:field 2 nil]
+ [:field "LIKED" {:base-type :type/Boolean}]
+ [:expression "T" {:base-type :type/Boolean}]]}}
+ (nest-expressions query))))))
diff --git a/test/metabase/xrays/automagic_dashboards/core_test.clj b/test/metabase/xrays/automagic_dashboards/core_test.clj
index 5cf71b585a8..51b30692727 100644
--- a/test/metabase/xrays/automagic_dashboards/core_test.clj
+++ b/test/metabase/xrays/automagic_dashboards/core_test.clj
@@ -1056,7 +1056,7 @@
(is (= []
(magic/candidate-tables db)))))))
-(deftest ^:parallel enhance-table-stats-test
+(deftest enhance-table-stats-test
(mt/with-temp [:model/Database {db-id :id} {}
:model/Table {table-id :id} {:db_id db-id}
:model/Field _ {:table_id table-id :semantic_type :type/PK}
@@ -1068,7 +1068,7 @@
(-> (#'magic/load-tables-with-enhanced-table-stats [[:= :id table-id]])
first)))))))
-(deftest ^:parallel enhance-table-stats-fk-test
+(deftest enhance-table-stats-fk-test
(mt/with-temp [:model/Database {db-id :id} {}
:model/Table {table-id :id} {:db_id db-id}
:model/Field _ {:table_id table-id :semantic_type :type/PK}