The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
nest-query => Lib (#64940)
Cam Saul15b31fb8342f
diff --git a/docs/developers-guide/driver-changelog.md b/docs/developers-guide/driver-changelog.mdindex 89fcdf8e2f8..645947c27ef 100644--- a/docs/developers-guide/driver-changelog.md+++ b/docs/developers-guide/driver-changelog.md@@ -48,6 +48,12 @@ title: Driver interface changelogyou can pass to [[metabase.lib.core/order-by]] -- for example a Lib `:metadata/column` or an [MBQL 5]`metabase.driver-api.core/order-by-clause`.+- `metabase.driver-api.core/nest-query` no longer automatically calls `metabase.driver-api.core/add-alias-info` on its+ results, but it also no longer expect its input to have this information. If you were using both of these tools in+ your driver, make change the order in which the are applied so `nest-query` happens first, followed by+ `add-alias-info`. Note that drivers deriving from `:sql` do not need to make any changes, since this is done by the+ base `:sql` driver implementation.+## Metabase 0.56.3- Added the driver multi-method `driver/describe-database*` that drivers should now implement instead of `driver/describe-database`.diff --git a/modules/drivers/sqlite/test/metabase/driver/sqlite_test.clj b/modules/drivers/sqlite/test/metabase/driver/sqlite_test.cljindex 64d018ca526..8a8d524a959 100644--- a/modules/drivers/sqlite/test/metabase/driver/sqlite_test.clj+++ b/modules/drivers/sqlite/test/metabase/driver/sqlite_test.clj@@ -214,14 +214,13 @@(deftest ^:parallel duplicate-identifiers-test(testing "Make sure duplicate identifiers (even with different cases) get unique aliases"(mt/test-driver :sqlite- (is (= '{:select [source.CATEGORY_2 AS CATEGORY_2+ (is (= '{:select [source.CATEGORY AS CATEGORYCOUNT (*) AS count]- :from [{:select [products.category AS category- products.category || ? AS CATEGORY_2]+ :from [{:select [products.category || ? AS CATEGORY]:from [products]}AS source]- :group-by [source.CATEGORY_2]- :order-by [source.CATEGORY_2 ASC]+ :group-by [source.CATEGORY]+ :order-by [source.CATEGORY ASC]:limit [1]}(sql.qp-test-util/query->sql-map(mt/mbql-query productsdiff --git a/src/metabase/driver/sql/query_processor.clj b/src/metabase/driver/sql/query_processor.cljindex 322d5b8d900..946e23658c0 100644--- a/src/metabase/driver/sql/query_processor.clj+++ b/src/metabase/driver/sql/query_processor.clj@@ -2045,8 +2045,8 @@[_driver inner-query](-> inner-querymaybe-nest-breakouts-in-queries-with-window-fn-aggregations- driver-api/add-alias-info- driver-api/nest-expressions))+ driver-api/nest-expressions+ driver-api/add-alias-info))(mu/defn mbql->honeysql :- [:or :map [:tuple [:= :inline] :map]]"Build the HoneySQL form we will compile to SQL and execute."diff --git a/src/metabase/driver_api/core.clj b/src/metabase/driver_api/core.cljindex da29b92cb50..20937bfc5c8 100644--- a/src/metabase/driver_api/core.clj+++ b/src/metabase/driver_api/core.clj@@ -48,7 +48,6 @@[metabase.query-processor.timezone :as qp.timezone][metabase.query-processor.util :as qp.util][metabase.query-processor.util.add-alias-info :as add]- [metabase.query-processor.util.nest-query :as nest-query][metabase.query-processor.util.relative-datetime :as qp.relative-datetime][metabase.query-processor.util.transformations.nest-breakouts :as qp.util.transformations.nest-breakouts][metabase.query-processor.writeback :as qp.writeback]@@ -140,8 +139,8 @@mdb/specmetabase.driver-api.impl/cachedmetabase.driver-api.impl/dispatch-by-clause-name-or-class+ metabase.driver-api.impl/nest-expressionsmi/instance-of?- nest-query/nest-expressionspremium-features/is-hosted?qp.compile/compileqp.debug/debug>diff --git a/src/metabase/driver_api/impl.clj b/src/metabase/driver_api/impl.cljindex 3436d25c384..e3b2b92bf7f 100644--- a/src/metabase/driver_api/impl.clj+++ b/src/metabase/driver_api/impl.clj@@ -1,11 +1,15 @@(ns metabase.driver-api.impl- {:clj-kondo/config '{:linters- ;; this is actually ok here since this is a drivers namespace- {:discouraged-namespace {metabase.query-processor.store {:level :off}}}}}+ {:clj-kondo/config+ '{:linters+ {:discouraged-namespace {metabase.legacy-mbql.util {:level :off}+ metabase.query-processor.store {:level :off}}+ :deprecated-namespace {:exclude [metabase.query-processor.store]}}}}(:require- ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.util :as mbql.u]+ [metabase.legacy-mbql.util :as mbql.u]+ [metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]- ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store]+ [metabase.query-processor.store :as qp.store]+ [metabase.query-processor.util.nest-query :as nest-query][metabase.util :as u][toucan2.core :as t2]))@@ -53,3 +57,17 @@(type x))))([x _](dispatch-by-clause-name-or-class x)))++(defn nest-expressions+ "Wrapper around [[metabase.query-processor.util.nest-query/nest-expressions]] to allow it to work with legacy MBQL+ inner queries. This is a temporary placeholder until we start migrating drivers to use Lib/MBQL 5 directly."+ [lib-query-or-legacy-inner-query]+ (if (:lib/type lib-query-or-legacy-inner-query)+ (nest-query/nest-expressions lib-query-or-legacy-inner-query)+ #_{:clj-kondo/ignore [:deprecated-var :discouraged-var]}+ (-> lib-query-or-legacy-inner-query+ (->> (lib/query-from-legacy-inner-query (qp.store/metadata-provider)+ (:id (lib.metadata/database (qp.store/metadata-provider)))))+ nest-query/nest-expressions+ lib/->legacy-MBQL+ :query)))diff --git a/src/metabase/lib/equality.cljc b/src/metabase/lib/equality.cljcindex 48c379e366e..e8966fca0f7 100644--- a/src/metabase/lib/equality.cljc+++ b/src/metabase/lib/equality.cljc@@ -104,46 +104,70 @@(sequential? x) ((get-method = :dispatch-type/sequential) x y):else (clojure.core/= x y)))-(defn- columns-equal-by-fn-when-non-nil-in-both [f col-1 col-2]+;; for debugging purposes the implementation of [[=]] for two columns return the "reason" things are not equal so we+;; can log this; [[=]] is basically just (not <reason>)++(defn- columns-not-equal-by-fn-when-non-nil-in-both+ [f col-1 col-2](let [v1 (f col-1)v2 (f col-2)]- (if (and v1 v2)- (clojure.core/= v1 v2)- true)))+ (when (and v1 v2 (not= v1 v2))+ f)))-(defn- columns-equal-by-fn [f col-1 col-2]- (clojure.core/= (f col-1) (f col-2)))+(defn- columns-not-equal-by-fn+ [f col-1 col-2]+ (when (not= (f col-1) (f col-2))+ f))(defn- ignore-default-temporal-bucket [bucket](when-not (clojure.core/= bucket :default)bucket))-(defmethod = :metadata/column+(defn- columns-not-equal-reason+ "Returns a keyword representing the reason why two columns fail an [[=]] check (for debugging purposes)."[col-1 col-2]- (and- ;; two column metadatas with different IDs are NEVER equal.- (columns-equal-by-fn-when-non-nil-in-both :id col-1 col-2)- ;; from the same source.- (columns-equal-by-fn-when-non-nil-in-both :lib/source col-1 col-2)- ;; same join alias- (columns-equal-by-fn lib.join.util/current-join-alias col-1 col-2)- ;;- ;; columns that don't have the same binning or temporal bucketing are never the same.- ;;- ;; same binning- (columns-equal-by-fn :metabase.lib.field/binning col-1 col-2)- ;; same bucketing- (columns-equal-by-fn (comp ignore-default-temporal-bucket lib.temporal-bucket/raw-temporal-bucket) col-1 col-2)- ;; check `:inherited-temporal-unit` as well if both columns have it.- (columns-equal-by-fn-when-non-nil-in-both (comp ignore-default-temporal-bucket :inherited-temporal-unit) col-1 col-2)- ;; finally make sure they have the same `:lib/source-column-alias` (if both columns have it) or `:name` (if for- ;; some reason they do not)+ (or+ ;; two column metadatas with different IDs are NEVER equal.+ (columns-not-equal-by-fn-when-non-nil-in-both :id col-1 col-2)+ ;; from the same source.+ (columns-not-equal-by-fn-when-non-nil-in-both :lib/source col-1 col-2)+ ;; same join alias+ (columns-not-equal-by-fn :metabase.lib.join/join-alias col-1 col-2)+ ;; same FK Field (for implicitly joined columns)+ (columns-not-equal-by-fn-when-non-nil-in-both :fk-field-id col-1 col-2)+ (columns-not-equal-by-fn :fk-join-alias col-1 col-2)+ ;; TODO (Cam 9/4/25) -- not super clear that this ought to be a reason for columns to be considered different since+ ;; `:fk-field-name` doesn't really seem to be super important... but this check seems to be needed, otherwise when+ ;; a there are multiple remappings from Col A => Col B (e.g. in a self-join) we'll potentially accidentally match+ ;; the wrong one. Maybe we can figure out a better way to make sure that doesn't happen.+ (columns-not-equal-by-fn-when-non-nil-in-both :fk-field-name col-1 col-2)+ ;;+ ;; columns that don't have the same binning or temporal bucketing are never the same.+ ;;+ ;; same binning+ (columns-not-equal-by-fn :metabase.lib.field/binning col-1 col-2)+ ;; same bucketing+ (when (columns-not-equal-by-fn (comp ignore-default-temporal-bucket lib.temporal-bucket/raw-temporal-bucket) col-1 col-2)+ 'temporal-bucket)+ ;; check `:inherited-temporal-unit` as well if both columns have it.+ (when (columns-not-equal-by-fn-when-non-nil-in-both (comp ignore-default-temporal-bucket :inherited-temporal-unit) col-1 col-2)+ :inherited-temporal-unit)+ ;; finally make sure they have the same `:lib/source-column-alias` (if both columns have it) or `:name` (if for+ ;; some reason they do not)(let [k (m/find-first (fn [k](and (k col-1)(k col-2)))[:lib/source-column-alias :name])](assert k "No key common to both columns")- (columns-equal-by-fn k col-1 col-2))))+ (columns-not-equal-by-fn k col-1 col-2))))++(defmethod = :metadata/column+ [col-1 col-2]+ (let [not-equal-reason (columns-not-equal-reason col-1 col-2)]+ (if not-equal-reason+ (log/debugf "Columns are not equal. Reason: %s" (pr-str not-equal-reason))+ (log/debug "Columns are equal."))+ (not not-equal-reason)))(mu/defn- resolve-field-id-in-source-card :- ::lib.schema.metadata/column"Integer Field ID: get metadata from the metadata provider. If this is the first stage of the query, merge indiff --git a/src/metabase/lib/field/resolution.cljc b/src/metabase/lib/field/resolution.cljcindex 729c92998da..461664b30b7 100644--- a/src/metabase/lib/field/resolution.cljc+++ b/src/metabase/lib/field/resolution.cljc@@ -608,6 +608,29 @@(-> (lib.expression/expression-metadata query stage-number expr)(assoc :lib/source-column-alias id-or-name))))))+(declare resolve-from-previous-stage-or-source)++(defn- resolve-nonexistent-deduplicated-column-name+ "Resolve a ref like `CATEGORY_2` to `CATEGORY` if the query only has the latter."+ [query stage-number id-or-name]+ (when (string? id-or-name)+ (when-let [[_match original-name suffix] (re-matches #"^(\w+)_([1-9][0-9]*)$" id-or-name)]+ (let [suffix (parse-long suffix)+ new-suffix (dec suffix)+ ;; e.g. `CATEGORY_3` becomes `CATEGORY_2`; `CATEGORY_2` becomes `CATEGORY`+ new-name (if (<= new-suffix 1)+ original-name+ (str original-name \_ new-suffix))]+ (log/debugf "Failed to resolve %s, trying to resolve Field %s instead..." (pr-str id-or-name) (pr-str new-name))+ (let [resolved (resolve-from-previous-stage-or-source query stage-number new-name)]+ (if (::fallback-metadata? resolved)+ (do+ (log/debugf "Failed to resolve %s as %s" (pr-str id-or-name) (pr-str new-name))+ nil)+ (do+ (log/debugf "Successfully resolved %s as %s" (pr-str id-or-name) (pr-str new-name))+ resolved)))))))+(mu/defn- resolve-from-previous-stage-or-source :- ::lib.metadata.calculation/visible-column[query :- ::lib.schema/querystage-number :- :int@@ -624,6 +647,9 @@;; try looking in the expressions in this stage to see if someone incorrectly used a field ref for an;; expression.(maybe-resolve-expression-in-current-stage query stage-number id-or-name)+ ;; if that fails and this is a deduplicated name like `CATEGORY_2` then try looking for `CATEGORY` and+ ;; so forth+ (resolve-nonexistent-deduplicated-column-name query stage-number id-or-name);; if we STILL can't find a match, return made-up fallback metadata.(fallback-metadata id-or-name))](when coldiff --git a/src/metabase/lib/walk.cljc b/src/metabase/lib/walk.cljcindex 795906def8e..9e4b3580ce3 100644--- a/src/metabase/lib/walk.cljc+++ b/src/metabase/lib/walk.cljc@@ -385,6 +385,21 @@clauses(range (count clauses)))))+(mu/defn walk-clauses-in-stage :- [:maybe ::lib.schema/stage.mbql]+ "Walk all the clauses in a stage. Calls++ (f clause)++ for every clause in the stage."+ [stage :- ::lib.schema/stage+ f :- [:=> [:cat :any] :any]]+ (when (lib.util/mbql-stage? stage)+ (reduce+ (fn [stage k]+ (m/update-existing stage k walk-clauses* f))+ stage+ [:aggregation :breakout :expressions :fields :filters :order-by])))+(mu/defn walk-clauses :- ::lib.schema/query"Walk all the MBQL clauses in a query in a depth-first manner.@@ -420,9 +435,4 @@(walk-clauses* f))))):lib.walk/stage- (when (lib.util/mbql-stage? stage-or-join)- (reduce- (fn [stage k]- (m/update-existing stage k walk-clauses* f))- stage-or-join- [:aggregation :breakout :expressions :fields :filters :order-by])))))))+ (walk-clauses-in-stage stage-or-join f))))))diff --git a/src/metabase/query_processor/util/nest_query.clj b/src/metabase/query_processor/util/nest_query.cljindex 0c2f479c3bf..7f8869b4d61 100644--- a/src/metabase/query_processor/util/nest_query.clj+++ b/src/metabase/query_processor/util/nest_query.clj@@ -1,352 +1,140 @@(ns metabase.query-processor.util.nest-query- "Utility functions for raising/nesting parts of MBQL queries. Currently, this only has [[nest-expressions]], but in- the future hopefully we can generalize this a bit so we can do more things that require us to introduce another- level of nesting, e.g. support window functions.+ "The [[nest-expressions]] query transformation (see docstring for more info).- (This namespace is here rather than in the shared MBQL lib because it relies on other QP-land utils like the QP- refs stuff.)"- (:refer-clojure :exclude [mapv select-keys some])+ TODO (Cam 10/22/25) -- this is a pure-MBQL-5 high-level query transformation, and almost certainly belongs in Lib+ rather than in QP -- we should move it there. (This also applies+ to [[metabase.query-processor.util.transformations.nest-breakouts]])."+ (:refer-clojure :exclude [select-keys some])(:require[medley.core :as m]- [metabase.api.common :as api]- ;; legacy usage -- don't use Legacy MBQL utils in QP code going forward, prefer Lib. This will be updated to use- ;; Lib only soon- ^{:clj-kondo/ignore [:discouraged-namespace]}- [metabase.legacy-mbql.schema :as mbql.s]- ^{:clj-kondo/ignore [:discouraged-namespace :deprecated-namespace]}- [metabase.legacy-mbql.util :as mbql.u][metabase.lib.core :as lib]- [metabase.lib.metadata :as lib.metadata]- [metabase.lib.schema.common :as lib.schema.common]+ [metabase.lib.equality :as lib.equality]+ [metabase.lib.schema :as lib.schema][metabase.lib.util.match :as lib.util.match]- [metabase.query-processor.middleware.annotate.legacy-helper-fns :as annotate.legacy-helper-fns]- ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store]- [metabase.query-processor.util.add-alias-info :as add]+ [metabase.lib.walk :as lib.walk][metabase.util :as u]- [metabase.util.log :as log][metabase.util.malli :as mu]- [metabase.util.performance :as perf :refer [mapv select-keys some]]))--(defn- all-fields-for-table [table-id]- (->> (lib.metadata/fields (qp.store/metadata-provider) table-id)- ;; The remove line is taken from the add-implicit-clauses middleware. It shouldn't be necessary, because any- ;; unused fields should be dropped from the inner query. It also shouldn't hurt anything, because the outer- ;; query shouldn't use these fields in the first place.- ;;- ;; TODO (Cam 9/12/25) -- this shouldn't be necessary anymore- (remove #(#{:sensitive :retired} (:visibility-type %)))- (map (fn [field]- [:field (u/the-id field) nil]))))--(defn- add-all-fields- "This adds all non-sensitive/retired fields (including fields with parent ids) to the passed-in query.-- The issue is that nest-query mostly relies on the preprocessor (specifically, the add-implicit-clauses middleware)- to add all possible fields to the newly created inner query. However, add-implicit-fields ignores fields with a- parent id. The main user of parent-id is mongo, and mongo doesn't use nest-query, so this usually isn't an- issue. However, bigquery also uses parent-id, and bigquery is a sql driver that uses nest-query. As a result,- without this function, nest-query wasn't adding bigquery struct member fields to the inner query, which caused sql- errors. This function adds in any missing fields and fixes those errors."- [{{source-table-id :source-table, :keys [fields]} :query, :as outer-query}]- (if source-table-id- (let [all-fields (all-fields-for-table source-table-id)- existing-fields (into #{} (map second) fields)]- (assoc-in outer-query [:query :fields]- (into fields- (remove (comp existing-fields second))- all-fields)))- outer-query))--(defn- remove-namespaced-options [options]- (when options- (not-empty (into {}- (remove (fn [[k _]]- (qualified-keyword? k)))- options))))--;;; TODO (Cam 6/16/25) -- duplicated with [[metabase.query-processor.middleware.resolve-joins/normalize-clause]], but-;;; this doesn't matter much since we will rewrite this whole namespace in MLv2 soon.-(defn- normalize-clause- "Normalize a `:field`/`:expression`/`:aggregation` clause by removing extra info so it can serve as a key for- `:qp/refs`. This removes `:source-field` if it is present -- don't use the output of this for anything but internal- key/distinct comparison purposes."- [clause]- (lib.util.match/match-one clause- ;; optimization: don't need to rewrite a `:field` clause without any options- [:field _ nil]- &match-- [:field id-or-name opts]- ;; this doesn't use [[mbql.u/update-field-options]] because this gets called a lot and the overhead actually adds up- ;; a bit- [:field id-or-name (remove-namespaced-options (cond-> (dissoc opts :source-field :effective-type)- (integer? id-or-name) (dissoc :base-type)))]-- ;; for `:expression` and `:aggregation` references, remove the options map if they are empty.- [:expression expression-name opts]- (if-let [opts (remove-namespaced-options opts)]- [:expression expression-name opts]- [:expression expression-name])-- [:aggregation index opts]- (if-let [opts (remove-namespaced-options opts)]- [:aggregation index opts]- [:aggregation index])-- _- &match))--(defn- joined-fields [inner-query]- (m/distinct-by- normalize-clause- (lib.util.match/match (perf/prewalk (fn [x]- (if (map? x)- (dissoc x :source-query :source-metadata :temporal-unit)- x))- inner-query)- [:field _ (_ :guard :join-alias)]- &match)))--(defn- keep-source+alias-props [field]- (update field 2 select-keys [::add/source-alias ::add/source-table :join-alias]))--(defn- nfc-root [[_ field-id]]- (when-let [field (and (int? field-id)- (lib.metadata/field (qp.store/metadata-provider) field-id))]- (when-let [nfc-root (first (:nfc-path field))]- {:table_id (:table-id field)- :name nfc-root})))--(defn- field-id-props [[_ field-id]]- (when-let [field (and (int? field-id)- (lib.metadata/field (qp.store/metadata-provider) field-id))]- {:table_id (:table-id field)- :name (:name field)}))--(defn- ->nominal-ref- "Transforms a ref into a simplified version of the form it would take as a nominal ref in a later stage.-- Nominal refs use the `desired-alias` as its `id-or-name`, and have `::add/source-table ::add/source` in the options.-- Other options (`:position`, `:join-alias`, other aliases) are dropped, since those are internal to the stage where the- column joined the party, not to later stages.-- Refs which are either missing `::add/desired-alias`, or which are coming directly from a table or join, are skipped.- We want to pick up the usages only, not anything from `:fields` lists."- [[_tag _id-or-name {::add/keys [source-alias source-table]}]]- (when (and source-alias- (= source-table ::add/source))- [:field source-alias {::add/source-table ::add/source}]))--(defn- remove-unused-fields [inner-query source]- (let [usages (lib.util.match/match inner-query #{:field :expression})- used-refs (into #{} (map keep-source+alias-props) usages)- nominal-refs (into #{} (keep ->nominal-ref) usages)- nfc-roots (into #{} (keep nfc-root) used-refs)]- (letfn [(used? [[_tag _id-or-name {::add/keys [source-table]}, :as a-ref]]- (or (contains? used-refs (keep-source+alias-props a-ref))- ;; We should also consider a Field to be used if we're referring to it with a nominal field literal- ;; ref in the next stage -- that's actually how you're supposed to be doing it anyway.- (and (= source-table ::add/source)- (contains? nominal-refs (->nominal-ref a-ref)))- (contains? nfc-roots (field-id-props a-ref))))- (used?* [a-ref]- (u/prog1 (used? a-ref)- (if <>- (log/debugf "Keeping used ref:\n%s" (u/pprint-to-str a-ref))- (log/debugf "Removing unused ref:\n%s" (u/pprint-to-str (keep-source+alias-props a-ref))))))- (remove-unused [refs]- (into []- (comp (filter used?*)- (m/distinct-by (fn [[_tag _id-or-name opts, :as _a-ref]]- (or (::add/desired-alias opts)- ;; if the field ref doesn't have a desired alias (probably because we- ;; added it in this namespace and haven't generated yet, then always- ;; keep it... just use a random UUID that should always be distinct- (random-uuid)))))- refs))]- (update source :fields remove-unused))))--(defn- append-join-fields- "This (supposedly) matches the behavior of [[metabase.lib.stage/add-cols-from-join]]. When we migrate this namespace- to Lib we can maybe use that."- [fields join-fields]- ;; we shouldn't consider different type info to mean two Fields are different even if everything else is the same. So- ;; give everything `:base-type` of `:type/*` (it will complain if we remove `:base-type` entirely from fields with a- ;; string name)- (letfn [(opts-signature [opts]- (not-empty- (merge- (u/select-non-nil-keys opts [:join-alias :binning])- ;; for purposes of deduplicating stuff, temporal unit = default is the same as not specifying temporal- ;; unit at all. Should that be part of normalization? Maybe, but there is some logic around adding default- ;; temporal bucketing that we don't do if `:default` is explicitly specified.- (when-let [temporal-unit (:temporal-unit opts)]- (when-not (= temporal-unit :default)- {:temporal-unit temporal-unit})))))- (ref-signature [[tag id-or-name opts, :as _ref]]- [tag id-or-name (opts-signature opts)])]- (into []- (comp cat- (m/distinct-by ref-signature))- [fields join-fields])))--(defn- append-join-fields-to-fields- "Add the fields from join `:fields`, if any, to the parent-level `:fields`."- [inner-query join-fields]- (cond-> inner-query- (seq join-fields) (update :fields append-join-fields join-fields)))--(mu/defn- nest-source :- ::mbql.s/SourceQuery- [inner-query :- ::mbql.s/SourceQuery]- (let [filter-clause (:filter inner-query)- keep-filter? (and filter-clause- (nil? (lib.util.match/match-one filter-clause :expression)))- source (-> inner-query- (select-keys [:source-table :source-query :source-metadata :joins :expressions])- ;; preprocess this in a superuser context so it's not subject to permissions checks. To get here in- ;; the first place we already had to do perms checks to make sure the query we're transforming is- ;; itself ok, so we don't need to run another check.- ;; (Not using mw.session/as-admin due to cyclic dependency.)- (as-> $source (binding [api/*is-superuser?* true]- ((requiring-resolve 'metabase.query-processor.preprocess/preprocess)- {:database (u/the-id (lib.metadata/database (qp.store/metadata-provider)))- :type :query- :query $source})))- lib/->legacy-MBQL- add-all-fields- add/add-alias-info- :query- (dissoc :limit)- (append-join-fields-to-fields (joined-fields inner-query))- (->> (remove-unused-fields inner-query))- (cond-> keep-filter?- (assoc :filter filter-clause)))]- (-> inner-query- (dissoc :source-table :source-metadata :joins)- (assoc :source-query source)- (cond-> keep-filter? (dissoc :filter)))))--(mu/defn- infer-expression-type :- [:maybe ::lib.schema.common/base-type]- [inner-query :- :map- expression :- [:maybe ::mbql.s/FieldOrExpressionDef]]- (when expression- #_{:clj-kondo/ignore [:deprecated-var]}- (let [mlv2-query (annotate.legacy-helper-fns/legacy-inner-query->mlv2-query inner-query)]- (lib/type-of mlv2-query (lib/->pMBQL expression)))))--(defn- raise-source-query-expression-ref- "Convert an `:expression` reference from a source query into an appropriate `:field` clause for use in the surrounding- query."- [{:keys [source-query], :as query} [_ expression-name opts :as _clause]]- (let [expression-definition #_{:clj-kondo/ignore [:deprecated-var]} (mbql.u/expression-with-name query expression-name)- base-type (infer-expression-type query expression-definition)- {::add/keys [desired-alias]} (lib.util.match/match-one source-query- [:expression (_ :guard (partial = expression-name)) source-opts]- source-opts)- source-alias (or desired-alias expression-name)]- [:field- source-alias- (-> opts- (assoc :base-type (or base-type :type/*)- ::add/source-table ::add/source- ::add/source-alias source-alias))]))--(defn- coerced-field?- [field-id]- (contains? (lib.metadata/field (qp.store/metadata-provider) field-id) :coercion-strategy))--(defn- coercible-field-ref?- [form]- (and (vector? form)- (let [[tag id-or-name opts] form]- (and (= tag :field)- (not (:qp/ignore-coercion opts))- (or (contains? opts :temporal-unit)- (and (int? id-or-name)- (coerced-field? id-or-name)))))))--(defn- rewrite-fields-and-expressions [query]- (lib.util.match/replace query- ;; don't rewrite anything inside any source queries or source metadata.- (_ :guard (constantly (some (partial contains? (set &parents))- [:source-query :source-metadata])))- &match-- :expression- (raise-source-query-expression-ref query &match)-- ;; Mark all Fields at the new top level as `:qp/ignore-coercion` so QP implementations know not to apply coercion- ;; or whatever to them a second time.- ;; In fact, we don't mark all Fields, only the ones we deem coercible. Marking all would make a bunch of tests- ;; fail, but it might still make sense. For example, #48721 would have been avoided by unconditional marking.- (_ :guard coercible-field-ref?)- (recur #_{:clj-kondo/ignore [:deprecated-var]}- (mbql.u/update-field-options &match assoc :qp/ignore-coercion true))-- [:field id-or-name (opts :guard :join-alias)]- (let [{::add/keys [desired-alias]} (lib.util.match/match-one (:source-query query)- [:field- (_ :guard (partial = id-or-name))- (matching-opts :guard #(= (:join-alias %) (:join-alias opts)))]- matching-opts)]- [:field id-or-name (cond-> opts- desired-alias (assoc ::add/source-alias desired-alias- ::add/desired-alias desired-alias))])-- ;; Some refs in the outer stage might be referring to these fields by `:name` (eg. ID) and not- ;; properly by their `desired-alias`/this ref's `source-alias` (eg. "People - User__ID")- ;; Since these refs are across stages, there's no need to adjust `:expression` refs.- [:field (id-or-name :guard string?) (opts :guard ::add/source-alias)]- [:field (::add/source-alias opts) opts]-- ;; when recursing into joins use the refs from the parent level.- (m :guard (every-pred map? :joins))- (let [{:keys [joins]} m]- (-> (dissoc m :joins)- rewrite-fields-and-expressions- (assoc :joins (mapv (fn [join]- (assoc join :qp/refs (:qp/refs query)))- joins))))))--(defn- should-nest-expressions?- "Whether we should nest the expressions in a inner query; true if-- 1. there are some expression definitions in the inner query, AND-- 2. there are some breakouts OR aggregations OR order-by in the inner query-- 3. AND the breakouts/aggregations/order-bys contain at least one `:expression` reference."- [{:keys [expressions], breakouts :breakout, aggregations :aggregation, order-bys :order-by, :as _inner-query}]- (and- ;; 1. has some expression definitions- (seq expressions)- ;; 2. has some breakouts or aggregations or order-by- (or (seq breakouts)- (seq aggregations)- (seq order-bys))- ;; 3. contains an `:expression` ref- (lib.util.match/match-one (concat breakouts aggregations order-bys)- :expression)))--(mu/defn- nest-expressions* :- ::mbql.s/SourceQuery- [inner-query :- ::mbql.s/SourceQuery]- (let [{:keys [expressions]- :as inner-query} (m/update-existing inner-query :source-query nest-expressions*)]- (if-not (should-nest-expressions? inner-query)- inner-query- (let [{:keys [source-query], :as inner-query} (nest-source inner-query)- inner-query (rewrite-fields-and-expressions inner-query)- source-query (assoc source-query :expressions expressions)]- (-> inner-query- (dissoc :source-query :expressions :expression-idents)- (assoc :source-query source-query))))))--(mu/defn nest-expressions :- ::mbql.s/SourceQuery- "Pushes the `:source-table`/`:source-query`, `:expressions`, and `:joins` in the top-level of the query into a- `:source-query` and updates `:expression` references and `:field` clauses with `:join-alias`es accordingly. See- tests for examples. This is used by the SQL QP to make sure expressions happen in a subselect."- [inner-query :- ::mbql.s/SourceQuery]- (-> inner-query- nest-expressions*- add/add-alias-info))+ [metabase.util.performance :refer [select-keys some]]))++;; Mark all Fields at the new top level as `:qp/ignore-coercion` so QP implementations know not to apply coercion+;; or whatever to them a second time.+;; In fact, we don't mark all Fields, only the ones we deem coercible. Marking all would make a bunch of tests+;; fail, but it might still make sense. For example, #48721 would have been avoided by unconditional marking.++(defn- contains-expression-refs? [location]+ (lib.util.match/match-one location :expression))++(defn- should-nest-expressions? [query path]+ (and (lib.walk/apply-f-for-stage-at-path lib/mbql-stage? query path)+ (some (fn [f]+ (contains-expression-refs? (lib.walk/apply-f-for-stage-at-path f query path)))+ [lib/breakouts+ lib/aggregations+ lib/order-bys])))++(def ^:private first-stage-keys+ #{:expressions :joins :source-table :source-card :filters})++;; TODO (Cam 10/22/25) -- somewhat duplicated with/copied+;; from [[metabase.query-processor.util.transformations.nest-breakouts/fields-used-in-breakouts-aggregations-or-expressions]]+(defn- fields-needed-by-second-stage [query path stage]+ (let [stage' (apply dissoc stage first-stage-keys)+ refs (volatile! (transient []))]+ ;; temporarily disable enforcement since this stage is technically invalid since we removed all+ ;; the [[first-stage-keys]]+ (binding [lib.schema/*HACK-disable-ref-validation* true]+ (lib.walk/walk-clauses-in-stage+ stage'+ (fn [clause]+ (u/prog1 clause+ (when (lib/clause-of-type? clause #{:field :expression})+ ;; preserve the unbucketed/unbinned versions of things in the new first stage; we'll apply the+ ;; binning/bucketing in the second stage instead+ (let [unbucketed-ref (-> clause+ (cond-> (lib/clause-of-type? clause :field) (lib/with-binning nil))+ (lib/with-temporal-bucket nil)+ ;; unset these just to be safe in case they were set -- they're technically+ ;; allowed if the binning/bucketing is happening at this stage but they're no+ ;; longer happening here so leaving them in place would be incorrect.+ (lib/update-options dissoc+ :metabase.lib.field/original-temporal-unit+ :original-temporal-unit+ :lib/original-binning))]+ (vswap! refs conj! unbucketed-ref)))))))+ (into+ []+ (comp (m/distinct-by (fn [[tag opts id-or-name]]+ [tag+ (select-keys opts [:join-alias])+ id-or-name]))+ ;; if you are a psycho and have both a Field ID ref and a Field name ref to the same column the we need to+ ;; deduplicate those otherwise this is going to result in queries with duplicate column+ ;; names (see [[literal-boolean-expressions-and-fields-in-conditions-test]])+ (m/distinct-by (fn [a-ref]+ (-> (lib.walk/apply-f-for-stage-at-path lib/metadata query path a-ref)+ ((juxt :lib/source-column-alias lib/current-join-alias))))))+ (persistent! @refs))))++(defn- new-first-stage [query path stage]+ (-> stage+ (select-keys (conj first-stage-keys :lib/type))+ (assoc :fields (lib/fresh-uuids (fields-needed-by-second-stage query path stage)))))++(defn- new-second-stage [query path stage]+ (let [returned-cols (lib.walk/apply-f-for-stage-at-path lib/returned-columns query path)]+ (letfn [(update-ref [a-ref]+ (let [col (lib.walk/apply-f-for-stage-at-path lib/metadata query path a-ref)+ unbucketed-col (-> col+ (lib/with-binning nil)+ (lib/with-temporal-bucket nil))+ returned-col (m/find-first (partial lib.equality/= unbucketed-col) returned-cols)]+ (-> returned-col+ lib/update-keys-for-col-from-previous-stage+ (lib/with-binning (lib/binning col))+ (lib/with-temporal-bucket (lib/raw-temporal-bucket col))+ lib/ref)))]+ ;; temporarily disable enforcement since this stage will be invalid while we're messing with it... it will look+ ;; pretty nice when we're done tho.+ (binding [lib.schema/*HACK-disable-ref-validation* true]+ (-> stage+ (as-> $stage (apply dissoc $stage first-stage-keys))+ (lib.walk/walk-clauses-in-stage+ (fn [clause]+ (cond-> clause+ (lib/clause-of-type? clause #{:field :expression})+ update-ref))))))))++(mu/defn- nest-expressions* :- [:sequential {:min 2, :max 2} ::lib.schema/stage.mbql]+ [query :- ::lib.schema/query+ path :- ::lib.walk/path+ stage :- ::lib.schema/stage.mbql]+ (let [new-first-stage (new-first-stage query path stage)]+ [new-first-stage+ (new-second-stage (assoc-in query path new-first-stage) path stage)]))++(mu/defn nest-expressions :- ::lib.schema/query+ "For queries with `expressions` in the final stage, adds an additional stage and moves the breakouts, aggregations,+ and order bys into the new final stage. This is because lots of our SQL databases don't really like when you do+ stuff like this:++ SELECT (x + ?) AS x_1+ FROM my_table+ GROUP BY (x + ?)+ ORDER BY (x + ?) ASC++ Why? They are dumb and can't figure out `x + ?` is the same thing. So instead we will introduce an additional stage+ that will give us SQL that looks like this:++ SELECT x_1 AS x_1+ FROM (+ SELECT (x + ?) AS x_1+ FROM my_table+ ) source+ GROUP BY x_1+ ORDER BY x_1 ASC"+ [query :- ::lib.schema/query]+ (lib.walk/walk-stages+ query+ (fn [query path stage]+ (when (should-nest-expressions? query path)+ (nest-expressions* query path stage)))))diff --git a/src/metabase/query_processor/util/transformations/nest_breakouts.clj b/src/metabase/query_processor/util/transformations/nest_breakouts.cljindex 07b427b348a..86b485d2355 100644--- a/src/metabase/query_processor/util/transformations/nest_breakouts.clj+++ b/src/metabase/query_processor/util/transformations/nest_breakouts.clj@@ -1,6 +1,6 @@(ns metabase.query-processor.util.transformations.nest-breakouts"TODO (Cam 8/7/25) -- this is a pure-MBQL-5 high-level query transformation, and almost certainly belongs in Lib- rather than in QP -- we should move it there."+ rather than in QP -- we should move it there. (This also applies to [[metabase.query-processor.util.nest-query]])."(:refer-clojure :exclude [mapv select-keys some])(:require[flatland.ordered.set :as ordered-set]@@ -71,23 +71,23 @@(def ^:private granularity{:time-unbucketed 0- :minute 1- :minute-of-hour 2- :hour 3- :hour-of-day 4- :day 5+ :minute 1+ :minute-of-hour 2+ :hour 3+ :hour-of-day 4+ :day 5:date-unbucketed 5- :day-of-week 6- :day-of-month 7- :day-of-year 8- :week 9- :week-of-year 10- :month 11- :month-of-year 12- :quarter 13+ :day-of-week 6+ :day-of-month 7+ :day-of-year 8+ :week 9+ :week-of-year 10+ :month 11+ :month-of-year 12+ :quarter 13:quarter-of-year 14- :year 15- :year-of-era 16})+ :year 15+ :year-of-era 16})(defn- original-temporal-unit[temporal-attributes]diff --git a/src/metabase/xrays/automagic_dashboards/combination.clj b/src/metabase/xrays/automagic_dashboards/combination.cljindex d8673ae110f..fdb7b704685 100644--- a/src/metabase/xrays/automagic_dashboards/combination.clj+++ b/src/metabase/xrays/automagic_dashboards/combination.clj@@ -88,23 +88,13 @@(let [model? (and (mi/instance-of? :model/Card source)(queries/model? source))mp (lib-be/application-database-metadata-provider database)- query (cond-> (lib/query mp (cond- (mi/instance-of? :model/Table source) (lib.metadata/table mp (u/the-id source))- (mi/instance-of? :model/Card source) (lib.metadata/card mp (u/the-id source))))- (:xrays/aggregations metric-definition)- (as-> $query (reduce lib/aggregate $query (:xrays/aggregations metric-definition)))-- (:xrays/breakouts metric-definition)- (as-> $query (reduce lib/breakout $query (:xrays/breakouts metric-definition)))-- (:xrays/filters metric-definition)- (as-> $query (reduce lib/filter $query (:xrays/filters metric-definition)))-- (not model?)- (as-> $query (reduce- lib/filter- $query- query-filter)))]+ query (as-> (lib/query mp (cond+ (mi/instance-of? :model/Table source) (lib.metadata/table mp (u/the-id source))+ (mi/instance-of? :model/Card source) (lib.metadata/card mp (u/the-id source)))) $query+ (reduce lib/aggregate $query (:xrays/aggregations metric-definition))+ (reduce lib/breakout $query (:xrays/breakouts metric-definition))+ (reduce lib/filter $query (:xrays/filters metric-definition))+ (reduce lib/filter $query (when (not model?) query-filter)))](assoc ground-metric-with-dimensions :dataset_query query)))(defn- instantiate-visualizationdiff --git a/src/metabase/xrays/automagic_dashboards/comparison.clj b/src/metabase/xrays/automagic_dashboards/comparison.cljindex 08bd000a048..cd64c20159c 100644--- a/src/metabase/xrays/automagic_dashboards/comparison.clj+++ b/src/metabase/xrays/automagic_dashboards/comparison.clj@@ -250,6 +250,10 @@"Create a comparison dashboard based on dashboard `dashboard` comparing subsets ofthe dataset defined by segments `left` and `right`."[dashboard left right opts]+ ;; disable ref validation because X-Rays does stuff in a wacko manner, it adds a bunch of filters and whatever that+ ;; use columns from joins before adding the joins themselves (same with expressions), which is technically invalid+ ;; at the time it happens but ends up resulting in a valid query at the end of the day. Maybe one day we can rework+ ;; this code to be saner(binding [lib.schema/*HACK-disable-ref-validation* true](let [opts (-> opts(m/update-existing-in [:left :cell-query] (partial lib/normalize ::ads/root.cell-query))diff --git a/src/metabase/xrays/automagic_dashboards/core.clj b/src/metabase/xrays/automagic_dashboards/core.cljindex 0808f4048a7..919c169314c 100644--- a/src/metabase/xrays/automagic_dashboards/core.clj+++ b/src/metabase/xrays/automagic_dashboards/core.clj@@ -831,6 +831,10 @@entitygetter-fnsetter-fn]+ ;; disable ref validation because X-Rays does stuff in a wacko manner, it adds a bunch of filters and whatever that+ ;; use columns from joins before adding the joins themselves (same with expressions), which is technically invalid+ ;; at the time it happens but ends up resulting in a valid query at the end of the day. Maybe one day we can rework+ ;; this code to be saner(binding [lib.schema/*HACK-disable-ref-validation* true](if-let [element-value (some-> entity :dataset_query not-empty getter-fn)](letfn [(splice-element [dashcard]@@ -905,6 +909,10 @@cell-url (format "%sadhoc/%s/cell/%s" public-endpoint(magic.util/encode-base64-json (:dataset_query query))(magic.util/encode-base64-json cell-query))]+ ;; disable ref validation because X-Rays does stuff in a wacko manner, it adds a bunch of filters and whatever+ ;; that use columns from joins before adding the joins themselves (same with expressions), which is technically+ ;; invalid at the time it happens but ends up resulting in a valid query at the end of the day. Maybe one day we+ ;; can rework this code to be saner(binding [lib.schema/*HACK-disable-ref-validation* true](query-based-analysis root opts(when cell-querydiff --git a/src/metabase/xrays/automagic_dashboards/populate.clj b/src/metabase/xrays/automagic_dashboards/populate.cljindex 4fb6e62932b..00658a6be21 100644--- a/src/metabase/xrays/automagic_dashboards/populate.clj+++ b/src/metabase/xrays/automagic_dashboards/populate.clj@@ -140,6 +140,10 @@{query :dataset_query, :keys [title description width height id] :as dashcard} :- ::ads/card-template[x y]](let [query-fields (when query+ ;; disable ref validation because X-Rays does stuff in a wacko manner, it adds a bunch of+ ;; filters and whatever that use columns from joins before adding the joins themselves (same+ ;; with expressions), which is technically invalid at the time it happens but ends up resulting+ ;; in a valid query at the end of the day. Maybe one day we can rework this code to be saner(binding [lib.schema/*HACK-disable-ref-validation* true](-> {:dataset_query (lib-be/normalize-query query)}queries/populate-card-query-fields@@ -320,6 +324,10 @@(mu/defn- create-dashboard-populate-dashcards :- [:sequential ::ads/dashcard][dashcards :- [:maybe [:sequential ::ads/card-template]]]+ ;; disable ref validation because X-Rays does stuff in a wacko manner, it adds a bunch of filters and whatever that+ ;; use columns from joins before adding the joins themselves (same with expressions), which is technically invalid+ ;; at the time it happens but ends up resulting in a valid query at the end of the day. Maybe one day we can rework+ ;; this code to be saner(let [card-id->can-run-adhoc-query (binding [lib.schema/*HACK-disable-ref-validation* true](into {}(map (juxt ::id :can_run_adhoc_query))diff --git a/test/metabase/driver/sql/query_processor_test.clj b/test/metabase/driver/sql/query_processor_test.cljindex b6a9b955866..ab88253f084 100644--- a/test/metabase/driver/sql/query_processor_test.clj+++ b/test/metabase/driver/sql/query_processor_test.clj@@ -15,6 +15,7 @@[metabase.lib.test-util :as lib.tu][metabase.lib.test-util.macros :as lib.tu.macros][metabase.lib.test-util.notebook-helpers :as lib.tu.notebook]+ [metabase.lib.test-util.places-cam-likes-metadata-provider :as lib.tu.places-cam-likes-metadata-provider][metabase.query-processor :as qp][metabase.query-processor.compile :as qp.compile][metabase.query-processor.middleware.limit :as limit]@@ -459,21 +460,16 @@DATE_TRUNC ("year" source.CREATED_AT) AS CREATED_ATsource.pivot-grouping AS pivot-groupingCOUNT (*) AS count]- :from [{:select [ORDERS.USER_ID AS USER_ID- ORDERS.PRODUCT_ID AS PRODUCT_ID- ORDERS.CREATED_AT AS CREATED_AT- ABS (0) AS pivot-grouping- ;; TODO: The order here is not deterministic! It's coming- ;; from [[metabase.query-processor.util.transformations.nest-breakouts]]- ;; or [[metabase.query-processor.util.nest-query]], which walks the query looking- ;; for refs in an arbitrary order, and returns `m/distinct-by` over that random- ;; order. Changing the map keys on the inner query can perturb this order; if you- ;; cause this test to fail based on shuffling the order of these joined fields- ;; just edit the expectation to match the new order. Tech debt issue: #39396- PRODUCTS__via__PRODUCT_ID.ID AS PRODUCTS__via__PRODUCT_ID__ID- PEOPLE__via__USER_ID.ID AS PEOPLE__via__USER_ID__ID- PRODUCTS__via__PRODUCT_ID.CATEGORY AS PRODUCTS__via__PRODUCT_ID__CATEGORY- PEOPLE__via__USER_ID.SOURCE AS PEOPLE__via__USER_ID__SOURCE]+ ;; TODO: The order here is not deterministic! It's coming+ ;; from [[metabase.query-processor.util.transformations.nest-breakouts]]+ ;; or [[metabase.query-processor.util.nest-query]], which walks the query looking for refs in an+ ;; arbitrary order, and returns `m/distinct-by` over that random order. Changing the map keys on the+ ;; inner query can perturb this order; if you cause this test to fail based on shuffling the order of+ ;; these joined fields just edit the expectation to match the new order. Tech debt issue: #39396+ :from [{:select [PRODUCTS__via__PRODUCT_ID.CATEGORY AS PRODUCTS__via__PRODUCT_ID__CATEGORY+ PEOPLE__via__USER_ID.SOURCE AS PEOPLE__via__USER_ID__SOURCE+ ORDERS.CREATED_AT AS CREATED_AT+ ABS (0) AS pivot-grouping]:from [ORDERS]:left-join [{:select [PRODUCTS.ID AS IDPRODUCTS.EAN AS EAN@@ -819,14 +815,13 @@(deftest ^:parallel expression-with-duplicate-column-name-test(testing "Can we use expression with same column name as table (#14267)"- (is (= '{:select [source.CATEGORY_2 AS CATEGORY_2+ (is (= '{:select [source.CATEGORY AS CATEGORYCOUNT (*) AS count]- :from [{:select [PRODUCTS.CATEGORY AS CATEGORY- CONCAT (PRODUCTS.CATEGORY ?) AS CATEGORY_2]+ :from [{:select [CONCAT (PRODUCTS.CATEGORY ?) AS CATEGORY]:from [PRODUCTS]}AS source]- :group-by [source.CATEGORY_2]- :order-by [source.CATEGORY_2 ASC]+ :group-by [source.CATEGORY]+ :order-by [source.CATEGORY ASC]:limit [1]}(-> (lib.tu.macros/mbql-query products{:expressions {:CATEGORY [:concat $category "2"]}@@ -1561,3 +1556,73 @@:query(->> (driver/prettify-native-form :h2))str/split-lines))))))++(deftest ^:parallel literal-boolean-expressions-and-fields-in-conditions-test+ (testing "mixing Field ID refs and Field Name refs to the same column should not result in broken queries"+ (let [true-value [:value true {:base_type :type/Boolean}]+ false-value [:value false {:base_type :type/Boolean}]+ mp lib.tu.places-cam-likes-metadata-provider/metadata-provider+ query (lib/query+ mp+ {:database 1+ :type :query+ :query {:expressions {"T" true-value, "F" false-value}+ :source-query {:source-table 1+ :fields [[:field 2 nil]]}+ :aggregation [[:count-where [:expression "T"]]+ [:count-where [:expression "F"]]+ ;; only a psycho would+ [:count-where [:field "LIKED" {:base-type :type/Boolean}]]+ [:count-where [:field 2 nil]]]+ :filter [:or+ [:field 2 nil]+ [:field "LIKED" {:base-type :type/Boolean}]+ [:expression "T"]]}})]+ (is (= ["SELECT"+ " SUM("+ " CASE"+ " WHEN \"source\".\"T\" THEN 1"+ " ELSE 0.0"+ " END"+ " ) AS \"count_where\","+ " SUM("+ " CASE"+ " WHEN \"source\".\"F\" THEN 1"+ " ELSE 0.0"+ " END"+ " ) AS \"count_where_2\","+ " SUM("+ " CASE"+ " WHEN \"source\".\"LIKED\" THEN 1"+ " ELSE 0.0"+ " END"+ " ) AS \"count_where_3\","+ " SUM("+ " CASE"+ " WHEN \"source\".\"LIKED\" THEN 1"+ " ELSE 0.0"+ " END"+ " ) AS \"count_where_4\""+ "FROM"+ " ("+ " SELECT"+ " TRUE AS \"T\","+ " FALSE AS \"F\","+ " \"source\".\"LIKED\" AS \"LIKED\""+ " FROM"+ " ("+ " SELECT"+ " \"PUBLIC\".\"PLACES\".\"LIKED\" AS \"LIKED\""+ " FROM"+ " \"PUBLIC\".\"PLACES\""+ " ) AS \"source\""+ " WHERE"+ " \"source\".\"LIKED\""+ " OR \"source\".\"LIKED\""+ " OR TRUE"+ " ) AS \"source\""]+ (-> query+ qp.compile/compile+ :query+ (->> (driver/prettify-native-form :h2))+ str/split-lines))))))diff --git a/test/metabase/lib/field/resolution_test.cljc b/test/metabase/lib/field/resolution_test.cljcindex a0a480f3777..11af689a02f 100644--- a/test/metabase/lib/field/resolution_test.cljc+++ b/test/metabase/lib/field/resolution_test.cljc@@ -1595,3 +1595,85 @@:lib/uuid "00000000-0000-0000-0000-000000000000":join-alias "Q1"}"O__ID"])))))++(deftest ^:parallel fallback-resolve-deduplicated-column-name-to-undeduplicated-name-test+ ;; see https://metaboat.slack.com/archives/C0645JP1W81/p1761241427398479 for more context+ (testing "If a query has something like `CATEGORY` but no `CATEGORY_2` we should resolve a `CATEGORY_2` ref to `CATEGORY`"+ (let [query (lib/query+ meta/metadata-provider+ {:lib/type :mbql/query+ :stages+ [{:lib/type :mbql.stage/mbql+ :source-table (meta/id :products)+ :expressions [[:concat+ {:lib/uuid "e190e161-cecf-4c33-aba7-767d3540b54c", :lib/expression-name "CATEGORY"}+ [:field+ {:lib/uuid "20c331ec-949c-4666-ad25-d09148d79614", :base-type :type/Text}+ (meta/id :products :category)]+ "2"]]+ :fields [[:expression+ {:base-type :type/Text, :lib/uuid "702e8a86-bd62-4e4c-8b38-5cd67c6d41f3"}+ "CATEGORY"]]}+ {:lib/type :mbql.stage/mbql+ :breakout [[:field+ {:lib/uuid "8d3e962d-1593-411d-99ed-c9fb30bf21c6", :base-type :type/Text}+ "CATEGORY"]]+ :aggregation [[:count {:lib/uuid "049bc679-bede-4cae-a28b-0a2b2a18ffd6"}]]+ :order-by [[:asc+ {:lib/uuid "4d859243-380d-4cef-9841-8edd0d90becc"}+ [:field+ {:lib/uuid "4e1391f8-3d4c-4c7c-a4cc-723cf15eb18a", :base-type :type/Text}+ "CATEGORY"]]]}]+ :database (meta/id)})+ expected (lib.field.resolution/resolve-field-ref+ query -1+ [:field {:lib/uuid "00000000-0000-0000-0000-000000000000" :base-type :type/Number} "CATEGORY"])]+ (is (= expected+ (lib.field.resolution/resolve-field-ref+ query -1+ [:field {:lib/uuid "00000000-0000-0000-0000-000000000000", :base-type :type/Number} "CATEGORY_2"])))+ (testing "Should be able to resolve nonexistent CATEGORY_3 to existent CATEGORY (i.e., try recursively)"+ (is (= expected+ (lib.field.resolution/resolve-field-ref+ query -1+ [:field {:lib/uuid "00000000-0000-0000-0000-000000000000", :base-type :type/Number} "CATEGORY_3"])))))))++(deftest ^:parallel fallback-resolve-deduplicated-column-name-to-undeduplicated-name-test-2+ (testing "Should be able to resolve nonexistent `CATEGORY_3` to existent `CATEGORY_2`"+ (let [query (lib/query+ meta/metadata-provider+ {:lib/type :mbql/query+ :stages+ [{:lib/type :mbql.stage/mbql+ :source-table (meta/id :products)+ :expressions [[:concat+ {:lib/uuid "e190e161-cecf-4c33-aba7-767d3540b54c", :lib/expression-name "CATEGORY"}+ [:field+ {:lib/uuid "20c331ec-949c-4666-ad25-d09148d79614", :base-type :type/Text}+ (meta/id :products :category)]+ "2"]]+ :fields [[:field+ {:base-type :type/Text, :lib/uuid "5a26b338-dbf2-4586-82f5-08d1eafde373"}+ "CATEGORY"]+ [:expression+ {:base-type :type/Text, :lib/uuid "702e8a86-bd62-4e4c-8b38-5cd67c6d41f3"}+ "CATEGORY"]]}+ {:lib/type :mbql.stage/mbql+ :breakout [[:field+ {:lib/uuid "8d3e962d-1593-411d-99ed-c9fb30bf21c6", :base-type :type/Text}+ "CATEGORY_2"]]+ :aggregation [[:count {:lib/uuid "049bc679-bede-4cae-a28b-0a2b2a18ffd6"}]]+ :order-by [[:asc+ {:lib/uuid "4d859243-380d-4cef-9841-8edd0d90becc"}+ [:field+ {:lib/uuid "4e1391f8-3d4c-4c7c-a4cc-723cf15eb18a", :base-type :type/Text}+ "CATEGORY_2"]]]}]+ :database (meta/id)})]+ (is (=? {:lib/deduplicated-name "CATEGORY_2"+ :lib/original-expression-name "CATEGORY"+ :lib/original-name "CATEGORY"+ :lib/source-column-alias "CATEGORY_2"+ ::lib.field.resolution/fallback-metadata? (symbol "nil #_\"key is not present.\"")}+ (lib.field.resolution/resolve-field-ref+ query -1+ [:field {:lib/uuid "00000000-0000-0000-0000-000000000000", :base-type :type/Number} "CATEGORY_3"]))))))diff --git a/test/metabase/lib/test_util/places_cam_likes_metadata_provider.cljc b/test/metabase/lib/test_util/places_cam_likes_metadata_provider.cljcnew file mode 100644index 00000000000..11f2fcbf2a6--- /dev/null+++ b/test/metabase/lib/test_util/places_cam_likes_metadata_provider.cljc@@ -0,0 +1,40 @@+(ns metabase.lib.test-util.places-cam-likes-metadata-provider+ (:require+ [metabase.lib.test-metadata :as meta]+ [metabase.lib.test-util :as lib.tu]))++(ns metabase.lib.test-util.places-cam-likes-metadata-provider)++(defn id+ ([]+ 1)+ ([table-name]+ (case table-name+ :places 1))+ ([table-name field-name]+ (case table-name+ :places (case field-name+ :id 1+ :liked 2))))++(def metadata-provider+ "Mock metadata provider with metadata similar to the `places-cam-likes` test dataset."+ (lib.tu/mock-metadata-provider+ {:database (merge meta/database+ {:id 1})+ :tables [(merge (meta/table-metadata :venues)+ {:id 1+ :db-id 1+ :name "PLACES"})]+ ;; TODO -- this metadata is a little incomplete, we can add better fingerprints and what not if we need it in+ ;; future tests.+ :fields [(merge (meta/field-metadata :venues :id)+ {:table-id 1+ :id 1})+ (merge (meta/field-metadata :venues :id)+ {:table-id 1+ :id 2+ :name "LIKED"+ :base-type :type/Boolean+ :effective-type :type/Boolean+ :fingerprint nil})]}))diff --git a/test/metabase/query_processor/util/nest_query_test.clj b/test/metabase/query_processor/util/nest_query_test.cljindex b6790b57c06..71afea782c7 100644--- a/test/metabase/query_processor/util/nest_query_test.clj+++ b/test/metabase/query_processor/util/nest_query_test.clj@@ -1,276 +1,155 @@(ns metabase.query-processor.util.nest-query-test(:require- [clojure.set :as set][clojure.test :refer :all]- [clojure.walk :as walk][metabase.driver :as driver][metabase.lib.core :as lib]- [metabase.lib.metadata :as lib.metadata][metabase.lib.test-metadata :as meta][metabase.lib.test-util :as lib.tu][metabase.lib.test-util.macros :as lib.tu.macros]+ [metabase.lib.test-util.places-cam-likes-metadata-provider :as lib.tu.places-cam-likes-metadata-provider][metabase.query-processor :as qp][metabase.query-processor.preprocess :as qp.preprocess]- ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.query-processor.util.add-alias-info :as add][metabase.query-processor.util.nest-query :as nest-query]- [metabase.test :as mt]- [metabase.util :as u]))+ [metabase.test :as mt]))-;;; TODO (Cam 7/18/25) -- update all the tests that use `with-temp` to use mock metadata providers instead.--;; TODO -- this is duplicated with [[metabase.query-processor.util.add-alias-info-test/remove-source-metadata]]-(defn- remove-source-metadata [x]- (walk/postwalk- (fn [x]- (if ((every-pred map? :source-metadata) x)- (dissoc x :source-metadata)- x))- x))--(defn- nest-expressions [query]+(defn- nest-expressions-mbql5 [query](driver/with-driver (or driver/*driver* :h2)(-> queryqp.preprocess/preprocess- lib/->legacy-MBQL- :query- nest-query/nest-expressions- remove-source-metadata)))+ nest-query/nest-expressions)))++(defn- nest-expressions [query]+ (-> query+ nest-expressions-mbql5+ lib/->legacy-MBQL+ :query))(deftest ^:parallel nest-expressions-test- (driver/with-driver :h2- (qp.store/with-metadata-provider meta/metadata-provider- (is (=? (lib.tu.macros/$ids venues- {:source-query {:source-table $$venues- :expressions {"double_price" [:* [:field %price {::add/source-table $$venues- ::add/source-alias "PRICE"- ::add/desired-alias "PRICE"}]- 2]}- :fields [[:field %price {::add/source-table $$venues- ::add/source-alias "PRICE"- ::add/desired-alias "PRICE"}]- [:expression "double_price" {::add/desired-alias "double_price"}]]}- :breakout [[:field %price {::add/source-table ::add/source- ::add/source-alias "PRICE"- ::add/desired-alias "PRICE"}]- [:field "double_price" {:base-type :type/Integer- ::add/source-table ::add/source- ::add/source-alias "double_price"- ::add/desired-alias "double_price"}]]- :aggregation [[:aggregation-options [:count] {::add/desired-alias "count"}]]- :order-by [[:asc [:field %price {::add/source-table ::add/source- ::add/source-alias "PRICE"- ::add/desired-alias "PRICE"}]]- [:asc [:field "double_price" {::add/source-table ::add/source- ::add/source-alias "double_price"- ::add/desired-alias "double_price"}]]]})- (-> (lib.tu.macros/mbql-query venues- {:expressions {"double_price" [:* $price 2]}- :breakout [$price- [:expression "double_price"]]- :aggregation [[:count]]})- qp.preprocess/preprocess- add/add-alias-info- nest-expressions))))))+ (is (=? (lib.tu.macros/$ids venues+ {:source-query {:source-table $$venues+ :expressions {"double_price" [:* [:field %price nil]+ 2]}+ :fields [[:field %price nil]+ [:expression "double_price" {}]]}+ :breakout [[:field "PRICE" {}]+ [:field "double_price" {}]]+ :aggregation [[:count]]+ :order-by [[:asc [:field "PRICE" {}]]+ [:asc [:field "double_price" {}]]]})+ (-> (lib/query+ meta/metadata-provider+ (lib.tu.macros/mbql-query venues+ {:expressions {"double_price" [:* $price 2]}+ :breakout [$price+ [:expression "double_price"]]+ :aggregation [[:count]]}))+ qp.preprocess/preprocess+ nest-expressions))))(deftest ^:parallel nest-order-by-expressions-test(testing "Expressions in an order-by clause result in nesting"- (driver/with-driver :h2- (qp.store/with-metadata-provider meta/metadata-provider- (is (partial= (lib.tu.macros/$ids venues- {:source-query {:source-table $$venues- :expressions {"double_price" [:* [:field %price {::add/source-table $$venues- ::add/source-alias "PRICE"- ::add/desired-alias "PRICE"}]- 2]}- :fields [[:field %price {::add/source-table $$venues- ::add/source-alias "PRICE"- ::add/desired-alias "PRICE"}]- [:expression "double_price" {::add/desired-alias "double_price"}]]}- :order-by [[:asc *double_price/Integer]]})- (-> (lib.tu.macros/mbql-query venues- {:expressions {"double_price" [:* $price 2]}- :fields [$price- [:expression "double_price"]]- :order-by [[:asc [:expression "double_price"]]]})- qp.preprocess/preprocess- add/add-alias-info- nest-expressions)))))))+ (is (=? (lib.tu.macros/$ids venues+ {:source-query {:source-table $$venues+ :expressions {"double_price" [:* [:field %price nil]+ 2]}+ :fields [[:field %price nil]+ [:expression "double_price" {}]]}+ :order-by [[:asc *double_price/Integer]]})+ (-> (lib/query+ meta/metadata-provider+ (lib.tu.macros/mbql-query venues+ {:expressions {"double_price" [:* $price 2]}+ :fields [$price+ [:expression "double_price"]]+ :order-by [[:asc [:expression "double_price"]]]}))+ qp.preprocess/preprocess+ nest-expressions)))))(deftest ^:parallel nest-order-by-literal-expressions-test(testing "Literal expressions in an order-by clause result in nesting"- (driver/with-driver :h2- (qp.store/with-metadata-provider meta/metadata-provider- (is (partial= (lib.tu.macros/$ids venues- {:source-query {:source-table $$venues- :expressions {"favorite" [:value "good venue" {:base_type :type/Text}]}- :fields [[:field %name {::add/source-table $$venues- ::add/source-alias "NAME"- ::add/desired-alias "NAME"}]- [:expression "favorite" {::add/desired-alias "favorite"}]]}- :filter [:= [:field %name {::add/source-table ::add/source- ::add/source-alias "NAME"- ::add/desired-alias "NAME"- :qp/ignore-coercion true}]- [:field "favorite" {:base-type :type/Text- ::add/source-table ::add/source- ::add/source-alias "favorite"- ::add/desired-alias "favorite"}]]- :order-by [[:asc *favorite/Text]]})- (-> (lib.tu.macros/mbql-query venues- {:expressions {"favorite" [:value "good venue" {:base_type :type/Text}]}- :fields [$name- [:expression "favorite"]]- :filter [:= $name [:expression "favorite"]]- :order-by [[:asc [:expression "favorite"]]]})- qp.preprocess/preprocess- add/add-alias-info- nest-expressions)))))))+ (is (=? (lib.tu.macros/$ids venues+ {:source-query {:source-table $$venues+ :expressions {"favorite" [:value "good venue" {:base_type :type/Text}]}+ :fields [[:field %name nil]+ [:expression "favorite" {}]]+ :filter [:=+ [:field %name nil]+ [:expression "favorite" {:base-type :type/Text}]]}+ :order-by [[:asc *favorite/Text]]})+ (-> (lib/query+ meta/metadata-provider+ (lib.tu.macros/mbql-query venues+ {:expressions {"favorite" [:value "good venue" {:base_type :type/Text}]}+ :fields [$name+ [:expression "favorite"]]+ :filter [:= $name [:expression "favorite"]]+ :order-by [[:asc [:expression "favorite"]]]}))+ qp.preprocess/preprocess+ nest-expressions)))))(deftest ^:parallel multiple-expressions-test(testing "Make sure the nested version of the query doesn't mix up expressions if we have ones that reference others"- (driver/with-driver :h2- (qp.store/with-metadata-provider meta/metadata-provider- (is (partial= (lib.tu.macros/$ids venues- {:source-query {:source-table $$venues- :expressions {"big_price"- [:+- [:field %price {::add/source-table $$venues- ::add/source-alias "PRICE"- ::add/desired-alias "PRICE"}]- 2]+ (is (=? (lib.tu.macros/$ids venues+ {:source-query {:source-table $$venues+ :expressions {"big_price"+ [:++ [:field %price nil]+ 2]- "my_cool_new_field"- [:/- [:field %price {::add/source-table $$venues- ::add/source-alias "PRICE"- ::add/desired-alias "PRICE"}]- [:expression "big_price" {::add/desired-alias "big_price"}]]}- :fields [[:field %id {::add/source-table $$venues- ::add/source-alias "ID"- ::add/desired-alias "ID"}]- [:field %price {::add/source-table $$venues- ::add/source-alias "PRICE"- ::add/desired-alias "PRICE"}]- [:expression "big_price" {::add/desired-alias "big_price"}]- [:expression "my_cool_new_field" {::add/desired-alias "my_cool_new_field"}]]}- :breakout [[:field "my_cool_new_field" {:base-type :type/Float- ::add/source-table ::add/source- ::add/source-alias "my_cool_new_field"- ::add/desired-alias "my_cool_new_field"}]]- :order-by [[:asc [:field %id {::add/source-table ::add/source- ::add/source-alias "ID"}]]]- :limit 3})- (-> (lib.tu.macros/mbql-query venues- {:expressions {"big_price" [:+ $price 2]- "my_cool_new_field" [:/ $price [:expression "big_price"]]}- :breakout [[:expression "my_cool_new_field"]]- :order-by [[:asc $id]]- :limit 3})- add/add-alias-info- nest-expressions)))))))+ "my_cool_new_field"+ [:/+ [:field %price nil]+ [:expression "big_price" {}]]}+ :fields [[:expression "my_cool_new_field" {}]+ [:field %id nil]]}+ :breakout [[:field "my_cool_new_field" {:base-type :type/Float}]]+ ;; uh... not sure ordering by ID is legal here??? Since it doesn't appear in the query results??+ :order-by [[:asc [:field "ID" {}]]+ [:asc [:field "my_cool_new_field" {:base-type :type/Float}]]]+ :limit 3})+ (-> (lib/query+ meta/metadata-provider+ (lib.tu.macros/mbql-query venues+ {:expressions {"big_price" [:+ $price 2]+ "my_cool_new_field" [:/ $price [:expression "big_price"]]}+ :breakout [[:expression "my_cool_new_field"]]+ :order-by [[:asc $id]]+ :limit 3}))+ nest-expressions)))))(deftest ^:parallel nest-expressions-ignore-source-queries-test(testing (str "When 'raising' :expression clauses, only raise ones in the current level. Handle duplicate expression ""names correctly.")- (driver/with-driver :h2- (qp.store/with-metadata-provider meta/metadata-provider- (let [query (lib.tu.macros/mbql-query venues- {:source-query {:source-table $$venues- :expressions {"x" [:* $price 2]}- :breakout [$id [:expression "x"]]}- :expressions {"x" [:* $price 4]}- :breakout [$id [:expression "x"]]- :limit 1})]- (mt/with-native-query-testing-context query- (is (partial= (lib.tu.macros/$ids venues- {:source-query {:source-query {:source-query {:source-table $$venues- :expressions {"x" [:*- [:field %price #::add{:source-table $$venues- :source-alias "PRICE"- :desired-alias "PRICE"}]- 2]}- :fields [[:field %id #::add{:source-table $$venues- :source-alias "ID"- :desired-alias "ID"}]- [:field %price #::add{:source-table $$venues- :source-alias "PRICE"- :desired-alias "PRICE"}]- [:expression "x" #::add{:desired-alias "x"}]]}- :breakout [[:field %id #::add{:source-table ::add/source- :source-alias "ID"- :desired-alias "ID"}]- [:field "x" {:base-type :type/Integer- ::add/source-table ::add/source- ::add/source-alias "x"- ::add/desired-alias "x"}]]}- :expressions {"x" [:*- [:field %price #::add{:source-table ::add/source- :source-alias "PRICE"}]- 4]}- :fields [[:field "ID" #::add{:source-table ::add/source- :source-alias "ID"- :desired-alias "ID"}]- [:field "x" {:base-type :type/Integer- ::add/source-table ::add/source- ::add/source-alias "x"- ::add/desired-alias "x"}]- [:expression "x" #::add{:desired-alias "x_2"}]]}- :breakout [[:field %id #::add{:source-table ::add/source- :source-alias "ID"- :desired-alias "ID"}]- [:field "x_2" {:base-type :type/Integer- ::add/source-table ::add/source- ::add/source-alias "x_2"- ::add/desired-alias "x_2"}]]- :limit 1})- (-> query add/add-alias-info nest-expressions)))))))))--(deftest ^:parallel nest-expressions-ignore-source-queries-from-joins-test- (testing "Ignores source-query from joins (#20809)"- (let [query {:source-table 2- :expressions {"CC" [:+ 1 1]}- :fields [[:field 33 {:join-alias "Question 4918"}]- [:field "count" {:join-alias "Question 4918"}]]- :joins [{:alias "Question 4918"- :strategy :left-join- :fields [[:field 33 {:join-alias "Question 4918"}]- [:field- "count"- {:join-alias "Question 4918"}]]- :condition [:=- [:field 5 nil]- [:field 33 {:join-alias "Question 4918"}]]- :source-card-id 4918- :source-query {:source-table 4- ;; nested query has filter values with join-alias that should not- ;; be selected- :filter [:=- [:field 26 {:join-alias "PRODUCTS__via__PRODUCT_ID"}]- [:value "Doohickey" {}]]- :aggregation [[:aggregation-options- [:count]- {:name "count"}]]- :breakout [[:field 33 nil]]- :limit 2- :order-by [[:asc- [:field 33 nil]]]- ;; nested query has an implicit join with conditions that should- ;; not be selected- :joins [{:alias "PRODUCTS__via__PRODUCT_ID"- :strategy :left-join- :condition [:=- [:field 33 nil]- [:field- 30- {:join-alias "PRODUCTS__via__PRODUCT_ID"}]]- :source-table 1- :fk-field-id 33}]}- :source-metadata [{:field_ref [:field 33 nil]}- {:field_ref [:aggregation 0]}]}]}]- (is (= [[:field 33 {:join-alias "Question 4918"}]- [:field "count" {:join-alias "Question 4918"}]]- (#'nest-query/joined-fields query))))))+ (let [query (lib/query+ meta/metadata-provider+ (lib.tu.macros/mbql-query venues+ {:source-query {:source-table $$venues+ :expressions {"x" [:* $price 2]}+ :breakout [$id [:expression "x"]]}+ :expressions {"x" [:* $price 4]}+ :breakout [$id [:expression "x"]]+ :limit 1}))]+ (mt/with-native-query-testing-context query+ (is (=? (lib.tu.macros/$ids venues+ {:source-query {:source-query {:source-query {:source-table $$venues+ :expressions {"x" [:*+ [:field %price nil]+ 2]}+ :fields [[:field %id nil]+ #_[:field %price nil]+ [:expression "x" {}]]}+ :breakout [[:field "ID" {}]+ [:field "x" {:base-type :type/Integer}]]}+ :expressions {"x" [:*+ [:field %price nil]+ 4]}+ :fields [[:field %id nil]+ [:expression "x" {}]]}+ :breakout [[:field "ID" {}]+ [:field "x" {:base-type :type/Integer}]]+ :limit 1})+ (nest-expressions query)))))))(deftest ^:parallel idempotence-test(testing "A nested query should return the same set of columns as the original"@@ -282,18 +161,20 @@:aggregation [[:count]];; filter on an implicit join:filter [:= $product-id->products.category "Doohickey"]})}]})- query (lib.tu.macros/mbql-query orders- {:joins [{:source-table "card__1"- :alias "Question 1"- :condition [:=- $product-id- [:field- %reviews.product-id- {:join-alias "Question 1"}]]- :fields :all}]- :expressions {"CC" [:+ 1 1]}- :limit 2})- nested (assoc query :query (qp.store/with-metadata-provider mp (nest-expressions query)))+ query (lib/query+ mp+ (lib.tu.macros/mbql-query orders+ {:joins [{:source-table "card__1"+ :alias "Question 1"+ :condition [:=+ $product-id+ [:field+ %reviews.product-id+ {:join-alias "Question 1"}]]+ :fields :all}]+ :expressions {"CC" [:+ 1 1]}+ :limit 2}))+ nested (assoc query :query (nest-expressions query))query* (lib/query mp query)nested* (lib/query mp nested)](is (= (map :lib/desired-column-alias (lib/returned-columns query*))@@ -308,16 +189,15 @@reviews{:breakout [$product_id]:aggregation [[:count]]- ;; filter on an implicit join+ ;; filter on an implicit join:filter [:= $product_id->products.category "Doohickey"]})}]})];; the result returned is not important, just important that the query is valid and completes- (is (vector?+ (is (seq(mt/rows(qp/process-query(lib/querymp- (mt/mbql-query- orders+ (mt/mbql-query orders{:joins [{:source-table "card__1":alias "Question 1":condition [:=@@ -329,520 +209,277 @@:expressions {"CC" [:+ 1 1]}:limit 2})))))))))-#_{:clj-kondo/ignore [:metabase/i-like-making-cams-eyes-bleed-with-horrifically-long-tests]}(deftest ^:parallel nest-expressions-with-joins-test- (driver/with-driver :h2- (qp.store/with-metadata-provider meta/metadata-provider- (testing "If there are any `:joins`, those need to be nested into the `:source-query` as well."- (is (partial= (lib.tu.macros/$ids venues- {:source-query {:source-table $$venues- :joins [{:strategy :left-join- :condition [:=- [:field %category-id {::add/source-table $$venues- ::add/source-alias "CATEGORY_ID"}]- [:field %category-id {:join-alias "CategoriesStats"- ::add/source-table "CategoriesStats"- ::add/source-alias "CATEGORY_ID"}]]- :source-query {:source-table $$venues- :aggregation [[:aggregation-options- [:max [:field %price {::add/source-table $$venues- ::add/source-alias "PRICE"}]]- {:name "MaxPrice"- ::add/desired-alias "MaxPrice"}]- [:aggregation-options- [:avg- [:field- %price- {::add/source-table $$venues- ::add/source-alias "PRICE"}]]- {:name "AvgPrice"- ::add/desired-alias "AvgPrice"}]- [:aggregation-options- [:min [:field %price {::add/source-table $$venues- ::add/source-alias "PRICE"}]]- {:name "MinPrice"- ::add/desired-alias "MinPrice"}]]- :breakout [[:field %category-id {::add/source-table $$venues- ::add/source-alias "CATEGORY_ID"- ::add/desired-alias "CATEGORY_ID"}]]- :order-by [[:asc [:field %category-id {::add/source-table $$venues- ::add/source-alias "CATEGORY_ID"- ::add/desired-alias "CATEGORY_ID"}]]]}- :alias "CategoriesStats"- :fields [[:field %category-id {:join-alias "CategoriesStats"}]- [:field "MaxPrice" {:join-alias "CategoriesStats"}]- [:field "AvgPrice" {:join-alias "CategoriesStats"}]- [:field "MinPrice" {:join-alias "CategoriesStats"}]]}]- :expressions {"RelativePrice" [:/- [:field %price {::add/source-table $$venues- ::add/source-alias "PRICE"- ::add/desired-alias "PRICE"}]- [:field "AvgPrice" {:base-type :type/Integer- :join-alias "CategoriesStats"- ::add/source-table "CategoriesStats"- ::add/source-alias "AvgPrice"- ::add/desired-alias "CategoriesStats__AvgPrice"}]]}- :fields [[:field %id {::add/source-table $$venues- ::add/source-alias "ID"- ::add/desired-alias "ID"}]- [:field %name {::add/source-table $$venues- ::add/source-alias "NAME"- ::add/desired-alias "NAME"}]- [:field %category-id {::add/source-table $$venues- ::add/source-alias "CATEGORY_ID"- ::add/desired-alias "CATEGORY_ID"}]- [:field %latitude {::add/source-table $$venues- ::add/source-alias "LATITUDE"- ::add/desired-alias "LATITUDE"}]- [:field %longitude {::add/source-table $$venues- ::add/source-alias "LONGITUDE"- ::add/desired-alias "LONGITUDE"}]- [:field %price {::add/source-table $$venues- ::add/source-alias "PRICE"- ::add/desired-alias "PRICE"}]- [:expression "RelativePrice" {::add/desired-alias "RelativePrice"}]- [:field %category-id {:join-alias "CategoriesStats"- ::add/source-table "CategoriesStats"- ::add/source-alias "CATEGORY_ID"- ::add/desired-alias "CategoriesStats__CATEGORY_ID"}]- [:field "MaxPrice" {:base-type :type/Integer- :join-alias "CategoriesStats"- ::add/source-table "CategoriesStats"- ::add/source-alias "MaxPrice"- ::add/desired-alias "CategoriesStats__MaxPrice"}]- [:field "AvgPrice" {:base-type :type/Float- :join-alias "CategoriesStats"- ::add/source-table "CategoriesStats"- ::add/source-alias "AvgPrice"- ::add/desired-alias "CategoriesStats__AvgPrice"}]- [:field "MinPrice" {:base-type :type/Integer- :join-alias "CategoriesStats"- ::add/source-table "CategoriesStats"- ::add/source-alias "MinPrice"- ::add/desired-alias "CategoriesStats__MinPrice"}]]}- :breakout [[:field %id {::add/source-table ::add/source- ::add/source-alias "ID"- ::add/desired-alias "ID"}]- [:field %name {::add/source-table ::add/source- ::add/source-alias "NAME"- ::add/desired-alias "NAME"}]- [:field %category-id {::add/source-table ::add/source- ::add/source-alias "CATEGORY_ID"- ::add/desired-alias "CATEGORY_ID"}]- [:field %latitude {::add/source-table ::add/source- ::add/source-alias "LATITUDE"- ::add/desired-alias "LATITUDE"}]- [:field %longitude {::add/source-table ::add/source- ::add/source-alias "LONGITUDE"- ::add/desired-alias "LONGITUDE"}]- [:field %price {::add/source-table ::add/source- ::add/source-alias "PRICE"- ::add/desired-alias "PRICE"}]- [:field "RelativePrice" {:base-type :type/Float- ::add/source-table ::add/source- ::add/source-alias "RelativePrice"- ::add/desired-alias "RelativePrice"}]- [:field %category-id {:join-alias "CategoriesStats"- ::add/source-alias "CategoriesStats__CATEGORY_ID"- ::add/desired-alias "CategoriesStats__CATEGORY_ID"- ::add/source-table ::add/source}]- [:field "MaxPrice" {:base-type :type/Integer- :join-alias "CategoriesStats"- ::add/source-alias "CategoriesStats__MaxPrice"- ::add/desired-alias "CategoriesStats__MaxPrice"- ::add/source-table ::add/source}]- [:field "AvgPrice" {:base-type :type/Integer- :join-alias "CategoriesStats"- ::add/source-alias "CategoriesStats__AvgPrice"- ::add/desired-alias "CategoriesStats__AvgPrice"- ::add/source-table ::add/source}]- [:field "MinPrice" {:base-type :type/Integer- :join-alias "CategoriesStats"- ::add/source-alias "CategoriesStats__MinPrice"- ::add/desired-alias "CategoriesStats__MinPrice"- ::add/source-table ::add/source}]]- :limit 3})- (-> (lib.tu.macros/mbql-query venues- {:breakout [$id- $name- $category-id- $latitude- $longitude- $price- [:expression "RelativePrice"]- &CategoriesStats.category-id- &CategoriesStats.*MaxPrice/Integer- &CategoriesStats.*AvgPrice/Integer- &CategoriesStats.*MinPrice/Integer]- :expressions {"RelativePrice" [:/ $price &CategoriesStats.*AvgPrice/Integer]}- :joins [{:strategy :left-join- :condition [:= $category-id &CategoriesStats.category-id]- :source-query {:source-table $$venues- :aggregation [[:aggregation-options [:max $price] {:name "MaxPrice"}]- [:aggregation-options [:avg $price] {:name "AvgPrice"}]- [:aggregation-options [:min $price] {:name "MinPrice"}]]- :breakout [$category-id]}- :alias "CategoriesStats"- :fields :all}]- :limit 3})- qp.preprocess/preprocess- add/add-alias-info- nest-expressions)))))))+ (testing "If there are any `:joins`, those need to be nested into the `:source-query` as well."+ (is (=? (lib.tu.macros/$ids venues+ {:source-query {:source-table $$venues+ :joins [{:strategy :left-join+ :condition [:=+ [:field %category-id nil]+ [:field %category-id {:join-alias "CategoriesStats"}]]+ :source-query {:source-table $$venues+ :aggregation [[:aggregation-options+ [:max [:field %price nil]]+ {:name "MaxPrice"}]+ [:aggregation-options+ [:avg+ [:field %price nil]]+ {:name "AvgPrice"}]+ [:aggregation-options+ [:min [:field %price nil]]+ {:name "MinPrice"}]]+ :breakout [[:field %category-id nil]]+ :order-by [[:asc [:field %category-id nil]]]}+ :alias "CategoriesStats"+ :fields [[:field %category-id {:join-alias "CategoriesStats"}]+ [:field "MaxPrice" {:join-alias "CategoriesStats"}]+ [:field "AvgPrice" {:join-alias "CategoriesStats"}]+ [:field "MinPrice" {:join-alias "CategoriesStats"}]]}]+ :expressions {"RelativePrice" [:/+ [:field %price nil]+ [:field "AvgPrice" {:join-alias "CategoriesStats"}]]}+ :fields [[:field %id nil]+ [:field %name nil]+ [:field %category-id nil]+ [:field %latitude nil]+ [:field %longitude nil]+ [:field %price nil]+ [:expression "RelativePrice" {}]+ [:field %category-id {:join-alias "CategoriesStats"}]+ [:field "MaxPrice" {:join-alias "CategoriesStats"}]+ [:field "AvgPrice" {:join-alias "CategoriesStats"}]+ [:field "MinPrice" {:join-alias "CategoriesStats"}]]}+ :breakout [[:field "ID" {}]+ [:field "NAME" {}]+ [:field "CATEGORY_ID" {}]+ [:field "LATITUDE" {}]+ [:field "LONGITUDE" {}]+ [:field "PRICE" {}]+ [:field "RelativePrice" {}]+ [:field "CategoriesStats__CATEGORY_ID" {}]+ [:field "CategoriesStats__MaxPrice" {}]+ [:field "CategoriesStats__AvgPrice" {}]+ [:field "CategoriesStats__MinPrice" {}]]+ :limit 3})+ (-> (lib/query+ meta/metadata-provider+ (lib.tu.macros/mbql-query venues+ {:breakout [$id+ $name+ $category-id+ $latitude+ $longitude+ $price+ [:expression "RelativePrice"]+ &CategoriesStats.category-id+ &CategoriesStats.*MaxPrice/Integer+ &CategoriesStats.*AvgPrice/Integer+ &CategoriesStats.*MinPrice/Integer]+ :expressions {"RelativePrice" [:/ $price &CategoriesStats.*AvgPrice/Integer]}+ :joins [{:strategy :left-join+ :condition [:= $category-id &CategoriesStats.category-id]+ :source-query {:source-table $$venues+ :aggregation [[:aggregation-options [:max $price] {:name "MaxPrice"}]+ [:aggregation-options [:avg $price] {:name "AvgPrice"}]+ [:aggregation-options [:min $price] {:name "MinPrice"}]]+ :breakout [$category-id]}+ :alias "CategoriesStats"+ :fields :all}]+ :limit 3}))+ qp.preprocess/preprocess+ nest-expressions)))))(deftest ^:parallel nest-expressions-eliminate-duplicate-coercion-test(testing "If coercion happens in the source query, don't do it a second time in the parent query (#12430)"- (driver/with-driver :h2- (qp.store/with-metadata-provider (lib.tu/merged-mock-metadata-provider- meta/metadata-provider- {:fields [{:id (meta/id :venues :price)- :coercion-strategy :Coercion/UNIXSeconds->DateTime- :effective-type :type/DateTime}]})- (is (=? (lib.tu.macros/$ids venues- {:source-query {:source-table $$venues- :expressions {"test" [:* 1 1]}- :fields [[:field %price {::add/source-table $$venues- ::add/source-alias "PRICE"- ::add/desired-alias "PRICE"}]- [:expression "test" {::add/desired-alias "test"}]]}- :breakout [[:field %price {:temporal-unit :day- :qp/ignore-coercion true- ::add/source-table ::add/source- ::add/source-alias "PRICE"- ::add/desired-alias "PRICE"}]- [:field "test" {:base-type :type/Integer- ::add/source-table ::add/source- ::add/source-alias "test"- ::add/desired-alias "test"}]]- :limit 1})- (-> (lib.tu.macros/mbql-query venues- {:expressions {"test" [:* 1 1]}- :breakout [$price- [:expression "test"]]- :limit 1})- add/add-alias-info- nest-expressions)))))))+ (let [mp (lib.tu/merged-mock-metadata-provider+ meta/metadata-provider+ {:fields [{:id (meta/id :venues :price)+ :coercion-strategy :Coercion/UNIXSeconds->DateTime+ :effective-type :type/DateTime}]})]+ (is (=? (lib.tu.macros/$ids venues+ {:source-query {:source-table $$venues+ :expressions {"test" [:* 1 1]}+ :fields [[:field %price {:temporal-unit (symbol "nil #_\"key is not present.\"")}]+ [:expression "test" {}]]}+ :breakout [[:field "PRICE" {:temporal-unit :day}]+ [:field "test" {}]]+ :limit 1})+ (-> (lib/query+ mp+ (lib.tu.macros/mbql-query venues+ {:expressions {"test" [:* 1 1]}+ :breakout [$price+ [:expression "test"]]+ :limit 1}))+ nest-expressions))))))(deftest ^:parallel multiple-joins-with-expressions-test(testing "We should be able to compile a complicated query with multiple joins and expressions correctly"- (driver/with-driver :h2- (mt/dataset test-data- (qp.store/with-metadata-provider meta/metadata-provider- (is (=? (:query (merge (let [products-category (lib.tu.macros/$ids orders- [:field %products.category {:join-alias "PRODUCTS__via__PRODUCT_ID"- ::add/source-table ::add/source- ::add/source-alias "PRODUCTS__via__PRODUCT_ID__CATEGORY"- ::add/desired-alias "PRODUCTS__via__PRODUCT_ID__CATEGORY"}])- created-at (lib.tu.macros/$ids orders- [:field %created-at {:temporal-unit :year- :qp/ignore-coercion true- ::add/source-table ::add/source- ::add/source-alias "CREATED_AT"- ::add/desired-alias "CREATED_AT"}])- pivot-grouping [:field "pivot-grouping" {:base-type :type/Float- ::add/source-table ::add/source- ::add/source-alias "pivot-grouping"- ::add/desired-alias "pivot-grouping"}]]- (lib.tu.macros/mbql-query orders- {:breakout [products-category created-at pivot-grouping]- :aggregation [[:aggregation-options [:count] {:name "count"- ::add/desired-alias "count"}]]- :order-by [[:asc products-category]- [:asc created-at]- [:asc pivot-grouping]]}))- (lib.tu.macros/mbql-query orders- {:source-query (let [product-id [:field %product-id {::add/source-table $$orders- ::add/source-alias "PRODUCT_ID"- ::add/desired-alias "PRODUCT_ID"}]- created-at [:field %created-at {::add/source-table $$orders- ::add/source-alias "CREATED_AT"- ::add/desired-alias "CREATED_AT"}]- pivot-grouping [:expression "pivot-grouping" {::add/desired-alias "pivot-grouping"}]- ;; TODO: The order here is not deterministic! It's coming- ;; from [[metabase.query-processor.util.transformations.nest-breakouts]]- ;; or [[metabase.query-processor.util.nest-query]], which walks- ;; the query looking for refs in an arbitrary order, and returns- ;; `m/distinct-by` over that random order. Changing the map keys- ;; on the inner query can perturb this order; if you cause this- ;; test to fail based on shuffling the order of these joined- ;; fields, just edit the expectation to match the new order.- ;; Tech debt issue: #39396- products-id [:field %products.id {:join-alias "PRODUCTS__via__PRODUCT_ID"- ::add/source-table "PRODUCTS__via__PRODUCT_ID"- ::add/source-alias "ID"- ::add/desired-alias "PRODUCTS__via__PRODUCT_ID__ID"}]- products-category [:field %products.category {:join-alias "PRODUCTS__via__PRODUCT_ID"- ::add/source-table "PRODUCTS__via__PRODUCT_ID"- ::add/source-alias "CATEGORY"- ::add/desired-alias "PRODUCTS__via__PRODUCT_ID__CATEGORY"}]]- {:source-table $$orders- :joins [{:source-query {:source-table $$products}- :alias "PRODUCTS__via__PRODUCT_ID"- :condition [:=- [:field %product-id {::add/source-table $$orders- ::add/source-alias "PRODUCT_ID"}]- [:field %products.id {:join-alias "PRODUCTS__via__PRODUCT_ID"- ::add/source-table "PRODUCTS__via__PRODUCT_ID"- ::add/source-alias "ID"}]]- :strategy :left-join- :fk-field-id %product-id}]- :expressions {"pivot-grouping" [:abs 0]}- :fields [product-id- created-at- pivot-grouping- products-id- products-category]})})))- (-> (lib.tu.macros/mbql-query orders- {:aggregation [[:aggregation-options [:count] {:name "count"}]]- :breakout [&PRODUCTS__via__PRODUCT_ID.products.category- !year.created-at- [:expression "pivot-grouping"]]- :expressions {"pivot-grouping" [:abs 0]}- :order-by [[:asc &PRODUCTS__via__PRODUCT_ID.products.category]- [:asc !year.created-at]- [:asc [:expression "pivot-grouping"]]]- :joins [{:source-table $$products- :strategy :left-join- :alias "PRODUCTS__via__PRODUCT_ID"- :fk-field-id %product-id- :condition [:= $product-id &PRODUCTS__via__PRODUCT_ID.products.id]}]})- add/add-alias-info- nest-expressions- ;; I'm tired of dealing with the nondeterministic order mentioned above, so just sort them by ID- ;; and call it a day for now.- (update-in [:source-query :fields] (fn [fields]- (concat- (take 3 fields)- (sort-by second (drop 3 fields)))))))))))))+ (is (=? (:query (merge (let [products-category (lib.tu.macros/$ids orders+ [:field %products.category {:join-alias "PRODUCTS__via__PRODUCT_ID"}])+ created-at (lib.tu.macros/$ids orders+ [:field %created-at {:temporal-unit :year+ :qp/ignore-coercion true}])+ pivot-grouping [:field "pivot-grouping" {:base-type :type/Float}]]+ (lib.tu.macros/mbql-query orders+ {:breakout [products-category+ created-at+ pivot-grouping]+ :aggregation [[:aggregation-options [:count] {:name "count"}]]+ :order-by [[:asc products-category]+ [:asc created-at]+ [:asc pivot-grouping]]}))+ (lib.tu.macros/mbql-query orders+ ;; TODO: The order here is not deterministic! It's coming+ ;; from [[metabase.query-processor.util.transformations.nest-breakouts]]+ ;; or [[metabase.query-processor.util.nest-query]], which walks the query looking for+ ;; refs in an arbitrary order, and returns `m/distinct-by` over that random order.+ ;; Changing the map keys on the inner query can perturb this order; if you cause this+ ;; test to fail based on shuffling the order of these joined fields, just edit the+ ;; expectation to match the new order. Tech debt issue: #39396+ {:source-query {:source-table $$orders+ :joins [{:source-query {:source-table $$products}+ :alias "PRODUCTS__via__PRODUCT_ID"+ :condition [:=+ [:field %product-id nil]+ [:field %products.id {:join-alias "PRODUCTS__via__PRODUCT_ID"}]]+ :strategy :left-join+ :fk-field-id %product-id}]+ :expressions {"pivot-grouping" [:abs 0]}+ :fields [[:field %products.category {:join-alias "PRODUCTS__via__PRODUCT_ID"}]+ [:field %orders.created-at nil]+ [:expression "pivot-grouping" {:base-type :type/Integer}]]}})))+ (-> (lib/query+ meta/metadata-provider+ (lib.tu.macros/mbql-query orders+ {:aggregation [[:aggregation-options [:count] {:name "count"}]]+ :breakout [&PRODUCTS__via__PRODUCT_ID.products.category+ !year.created-at+ [:expression "pivot-grouping"]]+ :expressions {"pivot-grouping" [:abs 0]}+ :order-by [[:asc &PRODUCTS__via__PRODUCT_ID.products.category]+ [:asc !year.created-at]+ [:asc [:expression "pivot-grouping"]]]+ :joins [{:source-table $$products+ :strategy :left-join+ :alias "PRODUCTS__via__PRODUCT_ID"+ :fk-field-id %product-id+ :condition [:= $product-id &PRODUCTS__via__PRODUCT_ID.products.id]}]}))+ nest-expressions)))))(deftest ^:parallel uniquify-aliases-test- (driver/with-driver :h2- (qp.store/with-metadata-provider meta/metadata-provider- (is (=? (lib.tu.macros/$ids products- {:source-query {:source-table $$products- :expressions {"CATEGORY" [:concat- [:field %category {::add/source-table $$products- ::add/source-alias "CATEGORY"- ::add/desired-alias "CATEGORY"}]- "2"]}- :fields [[:field %category {::add/source-table $$products- ::add/source-alias "CATEGORY"- ::add/desired-alias "CATEGORY"}]- [:expression "CATEGORY" {::add/desired-alias "CATEGORY_2"}]]}- :breakout [[:field "CATEGORY_2" {:base-type :type/Text- ::add/source-table ::add/source- ::add/source-alias "CATEGORY_2"- ::add/desired-alias "CATEGORY_2"}]]- :aggregation [[:aggregation-options [:count] {::add/desired-alias "count"}]]- :order-by [[:asc [:field "CATEGORY_2" {:base-type :type/Text- ::add/source-table ::add/source- ::add/source-alias "CATEGORY_2"- ::add/desired-alias "CATEGORY_2"}]]]- :limit 1})- (-> (lib.tu.macros/mbql-query products- {:expressions {"CATEGORY" [:concat $category "2"]}- :breakout [:expression "CATEGORY"]- :aggregation [[:count]]- :order-by [[:asc [:expression "CATEGORY"]]]- :limit 1})- qp.preprocess/preprocess- add/add-alias-info- lib/->legacy-MBQL- :query- nest-query/nest-expressions))))))+ (is (=? (lib.tu.macros/$ids products+ {:source-query {:source-table $$products+ :expressions {"CATEGORY" [:concat+ [:field %category nil]+ "2"]}+ :fields [[:expression "CATEGORY" {}]]}+ :breakout [[:field "CATEGORY" {:base-type :type/Text}]]+ :aggregation [[:count]]+ :order-by [[:asc [:field "CATEGORY" {:base-type :type/Text}]]]+ :limit 1})+ (-> (lib/query+ meta/metadata-provider+ (lib.tu.macros/mbql-query products+ {:expressions {"CATEGORY" [:concat $category "2"]}+ :breakout [:expression "CATEGORY"]+ :aggregation [[:count]]+ :order-by [[:asc [:expression "CATEGORY"]]]+ :limit 1}))+ nest-expressions))))(deftest ^:parallel uniquify-aliases-test-2- (driver/with-driver :h2- (mt/dataset test-data- (qp.store/with-metadata-provider meta/metadata-provider- (testing "multi-stage query with an expression name that matches a table column (#39059)"- (is (=? (lib.tu.macros/mbql-query orders- {:source-query {:breakout [[:field %id {}]- [:field %subtotal {}]- ;; Then exported as DISCOUNT from the middle layer.- [:field "DISCOUNT_2" {:base-type :type/Float- ::add/source-alias "DISCOUNT_2"- ::add/desired-alias "DISCOUNT_2"}]]- :source-query {:expressions {"DISCOUNT" [:coalesce [:field %discount {}] 0]}- :fields [[:field %id {::add/desired-alias "ID"}]- [:field %subtotal {::add/desired-alias "SUBTOTAL"}]- [:field %discount {::add/desired-alias "DISCOUNT"}]- ;; Exported as DISCOUNT_2 from this inner query.- [:expression "DISCOUNT"- {::add/desired-alias "DISCOUNT_2"}]]- :source-table $$orders}}- :source-query/model? true- :breakout [[:field "ID" {}]- [:field "SUBTOTAL" {}]- [:field "DISCOUNT" {:base-type :type/Float- ::add/source-alias "DISCOUNT"- ::add/desired-alias "DISCOUNT"}]]})- (-> (lib.tu.macros/mbql-query orders- {:source-query {:expressions {"DISCOUNT" [:coalesce $discount 0]}- :breakout [$id- $subtotal- [:expression "DISCOUNT"]]- :source-table $$orders}- :source-query/model? true- :breakout [[:field "ID" {:base-type :type/Integer}]- [:field "SUBTOTAL" {:base-type :type/Float}]- [:field "DISCOUNT" {:base-type :type/Float}]]})- qp.preprocess/preprocess- add/add-alias-info- lib/->legacy-MBQL- :query- nest-query/nest-expressions- (->> (assoc {:database (meta/id)- :type :query}- :query))))))))))--(defn- readable-query- "Attempt to make the results of [[add/add-alias-info]] and [[nest-query/nest-expressions]] a little less noisy so- they're actually readable/debuggable."- [query]- (letfn [(inner-query? [form]- (and (map? form)- ((some-fn :source-query :source-table) form)))- (inner-query [form]- (select-keys form [:source-query- :expressions- :breakout- :aggregation- :fields]))- (ref-options-map? [form]- (and (map? form)- (::add/desired-alias form)))- (table-symbol [table-id]- (let [table-name (:name (lib.metadata/table- (qp.store/metadata-provider)- table-id))]- (symbol (str "$$" (u/lower-case-en table-name)))))- (ref-options-map [form]- (-> (select-keys form [::add/source-table ::add/source-alias ::add/desired-alias :temporal-unit :bucketing :join-alias])- (set/rename-keys {::add/source-table :table, ::add/source-alias :source, ::add/desired-alias :desired})- (update :table (fn [table]- (if (integer? table)- (table-symbol table)- (if (= table ::add/source)- :source- table))))))- (field-ref? [form]- (and (vector? form)- (= (first form) :field)))- (field-symbol [field-id]- (let [field-name (:name (lib.metadata/field (qp.store/metadata-provider) field-id))]- (symbol (str \% (u/->kebab-case-en field-name)))))- (field-ref [form]- (let [[_tag id-or-name opts] form]- [:field- (cond-> id-or-name (pos-int? id-or-name) field-symbol)- opts]))]- (walk/postwalk- (fn [form]- (cond-> form- (inner-query? form) inner-query- (field-ref? form) field-ref- (ref-options-map? form) ref-options-map))- query)))+ (testing "multi-stage query with an expression name that matches a table column (#39059)"+ (is (=? (-> (lib.tu.macros/mbql-query orders+ {:source-query {:breakout [[:field "ID" {}]+ [:field "SUBTOTAL" {}]+ [:field "DISCOUNT" {}]]+ :source-query {:expressions {"DISCOUNT" [:coalesce [:field %discount nil] 0]}+ :fields [[:field %id nil]+ [:field %subtotal nil]+ [:expression "DISCOUNT" {}]]+ :source-table $$orders}}+ :breakout [[:field "ID" {}]+ [:field "SUBTOTAL" {}]+ [:field "DISCOUNT" {:base-type :type/Float}]]})+ :query)+ (-> (lib/query+ meta/metadata-provider+ (lib.tu.macros/mbql-query orders+ {:source-query {:expressions {"DISCOUNT" [:coalesce $discount 0]}+ :breakout [$id+ $subtotal+ [:expression "DISCOUNT"]]+ :source-table $$orders}+ :source-query/model? true+ :breakout [[:field "ID" {:base-type :type/Integer}]+ [:field "SUBTOTAL" {:base-type :type/Float}]+ [:field "DISCOUNT" {:base-type :type/Float}]]}))+ nest-expressions)))))(deftest ^:parallel do-not-remove-fields-when-referred-to-with-nominal-refs-test(testing "Don't remove fields if they are used in the next stage with a nominal field literal ref"- (qp.store/with-metadata-provider meta/metadata-provider- (driver/with-driver :h2- (let [query (lib.tu.macros/$ids products- {:source-query {:source-table $$products- :fields [[:field %id nil]- [:field %ean nil]- [:field %title nil]- [:field %category nil]- [:field %vendor nil]- [:field %price nil]- [:field %rating nil]- [:field %created-at {:temporal-unit :default}]]}- :expressions {"pivot-grouping" [:abs 0]}- :breakout [[:field "CATEGORY" {:base-type :type/Text}]- [:field "CREATED_AT" {:base-type :type/DateTime, :temporal-unit :month}]- [:expression "pivot-grouping"]]- :aggregation [[:aggregation-options [:count] {:name "count"}]]})- query' (add/add-alias-info query)]- (testing (str "with alias info:\n" (u/pprint-to-str (readable-query query')))- (is (=? '{:source-query {:source-query {:fields [[:field %id {}]- [:field %ean {}]- [:field %title {}]- [:field %category {}]- [:field %vendor {}]- [:field %price {}]- [:field %rating {}]- [:field %created-at {}]]}- :expressions {"pivot-grouping" [:abs 0]}- :fields [[:field "CATEGORY" {}]- [:field "CREATED_AT" {}]- [:expression "pivot-grouping" {}]]}- :breakout [[:field "CATEGORY" {}]- [:field "CREATED_AT" {}]- [:field "pivot-grouping" {}]]- :aggregation [[:aggregation-options [:count] {}]]}- (-> query'- nest-query/nest-expressions- readable-query)))))))))+ (let [query (lib/query+ meta/metadata-provider+ (lib.tu.macros/mbql-query products+ {:source-query {:source-table $$products+ :fields [[:field %id nil]+ [:field %ean nil]+ [:field %title nil]+ [:field %category nil]+ [:field %vendor nil]+ [:field %price nil]+ [:field %rating nil]+ [:field %created-at {:temporal-unit :default}]]}+ :expressions {"pivot-grouping" [:abs 0]}+ :breakout [[:field "CATEGORY" {:base-type :type/Text}]+ [:field "CREATED_AT" {:base-type :type/DateTime, :temporal-unit :month}]+ [:expression "pivot-grouping"]]+ :aggregation [[:aggregation-options [:count] {:name "count"}]]}))]+ (is (=? (lib.tu.macros/$ids products+ {:source-query {:source-query {:fields [[:field %id nil]+ [:field %ean nil]+ [:field %title nil]+ [:field %category nil]+ [:field %vendor nil]+ [:field %price nil]+ [:field %rating nil]+ [:field %created-at {:temporal-unit :default}]]}+ :expressions {"pivot-grouping" [:abs 0]}+ :fields [[:field "CATEGORY" {}]+ [:field "CREATED_AT" {}]+ [:expression "pivot-grouping" {}]]}+ :breakout [[:field "CATEGORY" {}]+ [:field "CREATED_AT" {}]+ [:field "pivot-grouping" {}]]+ :aggregation [[:aggregation-options [:count] {}]]})+ (nest-expressions query))))))(deftest ^:parallel nest-expressions-ignores-temporal-units-from-joined-fields(testing "clear temporal units from joined fields #48058"- (driver/with-driver :h2- (qp.store/with-metadata-provider meta/metadata-provider- (is (=? {:source-query- {:fields- [[:field- (meta/id :orders :user-id)- {::add/source-table (meta/id :orders)- ::add/source-alias "USER_ID"- ::add/desired-alias "USER_ID"}]- [:field- (meta/id :orders :total)- {::add/source-table (meta/id :orders)- ::add/source-alias "TOTAL"- ::add/desired-alias "TOTAL"}]- [:expression- "double_total"- {::add/desired-alias "double_total"}]- ;; TODO: The order here is not deterministic! It's coming- ;; from [[metabase.query-processor.util.transformations.nest-breakouts]]- ;; or [[metabase.query-processor.util.nest-query]], which walks the query looking for refs in an- ;; arbitrary order, and returns `m/distinct-by` over that random order. Changing the map keys on the- ;; inner query can perturb this order; if you cause this test to fail based on shuffling the order of- ;; these joined fields, just edit the expectation to match the new order. Tech debt issue: #39396- [:field- (meta/id :people :id)- {:join-alias "p"- ::add/source-table "p"- ::add/source-alias "ID"- ::add/desired-alias "p__ID"}]- [:field- (meta/id :people :created-at)- {:temporal-unit (symbol "nil #_\"key is not present.\"")- ::add/source-alias "CREATED_AT"- :join-alias "p"- ::add/desired-alias "p__CREATED_AT"- ::add/source-table "p"}]]}}- (-> (lib.tu.macros/mbql-query orders- {:expressions {"double_total" [:* $total 2]}- ;; this is a broken field ref! It should use the join alias `p`. Luckily- ;; the [[metabase.query-processor.middleware.fix-bad-field-id-refs]] middleware should fix it- ;; for us.- :breakout [!hour-of-day.people.created-at- [:expression "double_total"]]- :aggregation [[:count]]- :joins [{:source-table $$people- :alias "p"- :condition [:= $user-id &p.people.id]}]})- qp.preprocess/preprocess- add/add-alias-info- nest-expressions- ;; I'm tired of dealing with the nondeterministic order mentioned above, so just sort them by ID and- ;; call it a day for now.- (update-in [:source-query :fields] (fn [fields]- (concat- (take 3 fields)- (sort-by second (drop 3 fields))))))))))))+ ;; TODO: The order here is not deterministic! It's coming+ ;; from [[metabase.query-processor.util.transformations.nest-breakouts]]+ ;; or [[metabase.query-processor.util.nest-query]], which walks the query looking for refs in an arbitrary order+ ;; and returns `m/distinct-by` over that random order. Changing the map keys on the inner query can perturb this+ ;; order; if you cause this test to fail based on shuffling the order of these joined fields, just edit the+ ;; expectation to match the new order. Tech debt issue: #39396+ (is (=? {:source-query {:fields [[:field (meta/id :people :created-at) {:temporal-unit (symbol "nil #_\"key is not present.\"")+ :join-alias "p"}]+ [:expression "double_total" {}]]}}+ (-> (lib/query+ meta/metadata-provider+ (lib.tu.macros/mbql-query orders+ {:expressions {"double_total" [:* $total 2]}+ ;; this is a broken field ref! It should use the join alias `p`. Luckily+ ;; the [[metabase.query-processor.middleware.fix-bad-field-id-refs]] middleware should fix it+ ;; for us.+ :breakout [!hour-of-day.people.created-at+ [:expression "double_total"]]+ :aggregation [[:count]]+ :joins [{:source-table $$people+ :alias "p"+ :condition [:= $user-id &p.people.id]}]}))+ nest-expressions)))))(deftest ^:parallel wonky-breakout-test(let [mp (lib.tu/mock-metadata-provider@@ -866,63 +503,96 @@{:join-alias "model A - Product"}]]}]})}]})query (lib/querymp- {:database (meta/id)- :stages [{:lib/type :mbql.stage/mbql- :source-card 2- :expressions [[:abs {:lib/expression-name "pivot-grouping"} 0]]- :aggregation [[:sum {} [:field {:base-type :type/Number} "SUBTOTAL"]]]- :breakout [[:field {:base-type :type/Number, :join-alias "model A - Product"}- "Rating Bucket"]- [:expression {:base-type :type/Integer, :effective-type :type/Integer}- "pivot-grouping"]]}]- :lib/type :mbql/query})]- (qp.store/with-metadata-provider mp- (driver/with-driver :h2- (let [stages (-> query- qp.preprocess/preprocess- add/add-alias-info- lib/->legacy-MBQL- :query- nest-query/nest-expressions- (->> (lib/query-from-legacy-inner-query mp (meta/id)))- :stages- (->> (map (fn [stage]- (into []- (comp (mapcat stage)- (map lib/options)- (map ::add/desired-alias))- [:breakout :aggregation :fields])))))]- (is (= 3- (count stages)))- (testing "first stage (Card 1)"- (is (= ["ID"- "USER_ID"- "PRODUCT_ID"- "SUBTOTAL"- "TAX"- "TOTAL"- "DISCOUNT"- "CREATED_AT"- "QUANTITY"- "model A - Product__ID"- "model A - Product__EAN"- "model A - Product__TITLE"- "model A - Product__CATEGORY"- "model A - Product__VENDOR"- "model A - Product__PRICE"- "model A - Product__RATING"- "model A - Product__CREATED_AT"- "model A - Product__Rating Bucket"]- (nth stages 0))))- (testing "second stage (Card 2)"- (is (= ["ID"- "SUBTOTAL"- "CREATED_AT"- "model A - Product__Rating Bucket"- "pivot-grouping"]- (nth stages 1))))- (testing "third stage (original query)"- (is (= ["model A - Product__Rating Bucket"- "pivot-grouping"- "sum"]- (nth stages 2)))))))))+ (lib/query+ mp+ {:database (meta/id)+ :stages [{:lib/type :mbql.stage/mbql+ :source-card 2+ :expressions [[:abs {:lib/expression-name "pivot-grouping"} 0]]+ :aggregation [[:sum {} [:field {:base-type :type/Number} "SUBTOTAL"]]]+ :breakout [[:field {:base-type :type/Number, :join-alias "model A - Product"}+ "Rating Bucket"]+ [:expression {:base-type :type/Integer, :effective-type :type/Integer}+ "pivot-grouping"]]}]+ :lib/type :mbql/query}))+ stages (-> query+ nest-expressions-mbql5+ (as-> $query (driver/with-driver :h2+ (add/add-alias-info $query)))+ :stages+ (->> (map (fn [stage]+ (into []+ (comp (mapcat stage)+ (map lib/options)+ (map ::add/desired-alias))+ [:breakout :aggregation :fields])))))]+ (is (= 3+ (count stages)))+ (testing "first stage (Card 1)"+ (is (= ["ID"+ "USER_ID"+ "PRODUCT_ID"+ "SUBTOTAL"+ "TAX"+ "TOTAL"+ "DISCOUNT"+ "CREATED_AT"+ "QUANTITY"+ "model A - Product__ID"+ "model A - Product__EAN"+ "model A - Product__TITLE"+ "model A - Product__CATEGORY"+ "model A - Product__VENDOR"+ "model A - Product__PRICE"+ "model A - Product__RATING"+ "model A - Product__CREATED_AT"+ "model A - Product__Rating Bucket"]+ (nth stages 0))))+ (testing "second stage (Card 2)"+ (is (= ["SUBTOTAL"+ "model A - Product__Rating Bucket"+ "pivot-grouping"]+ (nth stages 1))))+ (testing "third stage (original query)"+ (is (= ["model A - Product__Rating Bucket"+ "pivot-grouping"+ "sum"]+ (nth stages 2))))))++(deftest ^:parallel literal-boolean-expressions-and-fields-in-conditions-test+ (testing "mixing Field ID refs and Field Name refs to the same column should not result in broken queries"+ (let [true-value [:value true {:base_type :type/Boolean}]+ false-value [:value false {:base_type :type/Boolean}]+ mp lib.tu.places-cam-likes-metadata-provider/metadata-provider+ query (lib/query+ mp+ {:database 1+ :type :query+ :query {:expressions {"T" true-value, "F" false-value}+ :source-query {:source-table 1+ :fields [[:field 2 nil]]}+ :aggregation [[:count-where [:expression "T"]]+ [:count-where [:expression "F"]]+ ;; only a true psycho would do this+ [:count-where [:field "LIKED" {:base-type :type/Boolean}]]+ [:count-where [:field 2 nil]]]+ :filter [:or+ [:field 2 nil]+ [:field "LIKED" {:base-type :type/Boolean}]+ [:expression "T"]]}})]+ (is (= {:aggregation [[:count-where [:field "T" {:base-type :type/Boolean}]]+ [:count-where [:field "F" {:base-type :type/Boolean}]]+ [:count-where [:field "LIKED" {:base-type :type/Boolean}]]+ [:count-where [:field "LIKED" {:base-type :type/Boolean}]]]+ :source-query {:source-query {:fields [[:field 2 nil]], :source-table 1}+ :expressions {"F" [:value false {:base_type :type/Boolean}]+ "T" [:value true {:base_type :type/Boolean}]}+ :fields [[:expression "T" {:base-type :type/Boolean}]+ [:expression "F" {:base-type :type/Boolean}]+ ;; make sure we do not include duplicate entries for `LIKED`/Field 2+ [:field "LIKED" {:base-type :type/Boolean}]]+ :filter [:or+ [:field 2 nil]+ [:field "LIKED" {:base-type :type/Boolean}]+ [:expression "T" {:base-type :type/Boolean}]]}}+ (nest-expressions query))))))diff --git a/test/metabase/xrays/automagic_dashboards/core_test.clj b/test/metabase/xrays/automagic_dashboards/core_test.cljindex 5cf71b585a8..51b30692727 100644--- a/test/metabase/xrays/automagic_dashboards/core_test.clj+++ b/test/metabase/xrays/automagic_dashboards/core_test.clj@@ -1056,7 +1056,7 @@(is (= [](magic/candidate-tables db)))))))-(deftest ^:parallel enhance-table-stats-test+(deftest enhance-table-stats-test(mt/with-temp [:model/Database {db-id :id} {}:model/Table {table-id :id} {:db_id db-id}:model/Field _ {:table_id table-id :semantic_type :type/PK}@@ -1068,7 +1068,7 @@(-> (#'magic/load-tables-with-enhanced-table-stats [[:= :id table-id]])first)))))))-(deftest ^:parallel enhance-table-stats-fk-test+(deftest enhance-table-stats-fk-test(mt/with-temp [:model/Database {db-id :id} {}:model/Table {table-id :id} {:db_id db-id}:model/Field _ {:table_id table-id :semantic_type :type/PK}