The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
resolve-joins middleware => Lib, remove add-source-metadata (#63608)
Cam Sauldec95aabe733
diff --git a/enterprise/backend/src/metabase_enterprise/advanced_permissions/query_processor/middleware/permissions.clj b/enterprise/backend/src/metabase_enterprise/advanced_permissions/query_processor/middleware/permissions.cljindex 978a29c20e1..caddd68d99f 100644--- a/enterprise/backend/src/metabase_enterprise/advanced_permissions/query_processor/middleware/permissions.clj+++ b/enterprise/backend/src/metabase_enterprise/advanced_permissions/query_processor/middleware/permissions.clj@@ -4,7 +4,6 @@[metabase.api.common :as api][metabase.lib.core :as lib][metabase.lib.schema :as lib.schema]- [metabase.lib.walk :as lib.walk][metabase.permissions.core :as perms][metabase.premium-features.core :refer [defenterprise-schema]][metabase.query-permissions.core :as query-perms]@@ -29,23 +28,10 @@[{database-id :database, :as _query}](perms/native-download-permission-for-user api/*current-user-id* database-id))-;;; TODO (Cam 9/11/25) -- move this to `lib.walk.util` once my PR that adds it finally gets reviewed-(mu/defn- any-native-stage?- "Returns true if any stage of this query is native."- [query :- ::lib.schema/query]- (let [has-native-stage? (atom false)]- (lib.walk/walk-stages- query- (fn [_query _path stage]- (when (= (:lib/type stage) :mbql.stage/native)- (reset! has-native-stage? true))- nil))- @has-native-stage?))-(mu/defmethod current-user-download-perms-level :mbql.stage/mbql[{db-id :database, :as query} :- ::lib.schema/query](let [{:keys [table-ids native?]} (query-perms/query->source-ids query)- perms (if (or native? (any-native-stage? query))+ perms (if (or native? (lib/any-native-stage? query));; If we detect any native subqueries/joins, even with source-card IDs, require full native;; download perms#{(perms/native-download-permission-for-user api/*current-user-id* db-id)}diff --git a/src/metabase/lib/core.cljc b/src/metabase/lib/core.cljcindex a2c6a63420c..4ee5fdb65ac 100644--- a/src/metabase/lib/core.cljc+++ b/src/metabase/lib/core.cljc@@ -154,6 +154,7 @@[lib.convert->legacy-MBQL->pMBQL+ legacy-default-join-aliaswithout-cleaning][metabase.lib.convert.metadata-to-legacylib-metadata-column->legacy-metadata-column@@ -444,6 +445,8 @@source-table-idupdate-query-stage][metabase.lib.walk.util+ all-field-idsall-source-card-idsall-source-table-ids- all-template-tags])+ all-template-tags+ any-native-stage?])diff --git a/src/metabase/lib/field/resolution.cljc b/src/metabase/lib/field/resolution.cljcindex d456892007e..69dc15f28bd 100644--- a/src/metabase/lib/field/resolution.cljc+++ b/src/metabase/lib/field/resolution.cljc@@ -84,30 +84,30 @@:lib/original-display-name (:display-name col))(->> (add-parent-column-metadata metadata-providerable)))))-(mu/defn- column-with-name :- [:maybe ::lib.schema.metadata/column]- [columns :- [:sequential ::lib.schema.metadata/column]- column-name :- :string]- ;; look for a match with the same `desired-column-alias`; if that fails, look for a match with using+(mu/defn- column-with-name-in-previous-stage-returned-columns :- [:maybe ::lib.schema.metadata/column]+ [previous-stage-cols :- ::lib.metadata.calculation/returned-columns+ column-name :- :string]+ ;; look for a match with the same `desired-colum-alias`; if that fails, look for a match with using;; legacy `deduplicated-name`(some (fn [k](m/find-first #(= (k %) column-name)- columns))+ previous-stage-cols))[:lib/desired-column-alias:lib/deduplicated-name]))-(mu/defn- resolve-in-previous-stage-metadata-without-updating-keys :- [:maybe ::lib.schema.metadata/column]+(mu/defn- resolve-in-previous-stage-returned-columns-without-updating-keys :- [:maybe ::lib.metadata.calculation/returned-column]"Find the matching column metadata in `cols` for `id-or-name`.This metadata should be relative to the previous stage, or join, or whatever!!!! It should not be relative to thecurrent stage."[metadata-providerable :- ::lib.schema.metadata/metadata-providerable- cols :- [:sequential ::lib.schema.metadata/column]+ previous-stage-cols :- [:maybe ::lib.metadata.calculation/returned-columns]id-or-name :- ::id-or-name](letfn [(resolve* [id-or-name](if (string? id-or-name)- (column-with-name cols id-or-name)+ (column-with-name-in-previous-stage-returned-columns previous-stage-cols id-or-name);; `id-or-name` is an ID- (or (m/find-first #(= (:id %) id-or-name) cols)+ (or (m/find-first #(= (:id %) id-or-name) previous-stage-cols)(do(log/debugf "Failed to find column in metadata with ID %s" (pr-str id-or-name))(when-some [field (lib.metadata/field metadata-providerable id-or-name)]@@ -125,7 +125,7 @@(log/debugf "Failed to find match for %s. Found:\n%s"(pr-str id-or-name)(u/pprint-to-str (map #(select-keys % [:id :lib/desired-column-alias :lib/deduplicated-name])- cols)))))))+ previous-stage-cols)))))))(def ^:private opts-propagated-keys"Keys to copy non-nil values directly from `:field` opts into column metadata."@@ -252,6 +252,8 @@:fingerprint:visibility-type})+(declare resolve-in-previous-stage-returned-columns-and-update-keys)+(mu/defn- additional-metadata-from-source-card :- [:maybe :map]"Calculate additional metadata to include from a source model or Card for an already-resolved column."[query :- ::lib.schema/query@@ -266,29 +268,28 @@;; if we have `id` then filter out anything that is definitely not a;; match(:id col) (filter #(= (:id %) (:id col)))))]- (let [card-cols (for [col card-cols]- (-> col- lib.field.util/update-keys-for-col-from-previous-stage- (assoc :lib/source :source/card, :lib/card-id card-id)))]- ;; prefer resolution with `:lib/source-column-alias` over `:id` if we have it because it will be- ;; unique/unambiguous if multiple versions of the column (e.g. with different bucketing units) are- ;; returned- (when-some [col (resolve-in-previous-stage-metadata-without-updating-keys query card-cols (:lib/source-column-alias col))]- (let [propagated-keys (if (= (:type card) :model)- model-propagated-keys- regular-card-propagated-keys)]- (select-keys col propagated-keys)))))))+ ;; prefer resolution with `:lib/source-column-alias` over `:id` if we have it because it will be+ ;; unique/unambiguous if multiple versions of the column (e.g. with different bucketing units) are+ ;; returned+ (when-some [col (resolve-in-previous-stage-returned-columns-and-update-keys query card-cols (:lib/source-column-alias col))]+ (let [col (assoc col :lib/source :source/card, :lib/card-id card-id)+ propagated-keys (if (= (:type card) :model)+ model-propagated-keys+ regular-card-propagated-keys)]+ (select-keys col propagated-keys))))))(:qp/stage-is-from-source-card stage)(let [card-id (:qp/stage-is-from-source-card stage)]{:lib/card-id card-id}))))-(mu/defn- resolve-in-previous-stage-metadata-and-update-keys :- [:maybe ::lib.metadata.calculation/visible-column]+(mu/defn- resolve-in-previous-stage-returned-columns-and-update-keys :- [:maybe ::lib.metadata.calculation/visible-column]+ "Resolves by `:id` = `id-or-name`, or `:lib/desired-column-alias` in the previous stage returned columns (=+ `:lib/source-column-alias` in the current stage) = `id-or-name`."[query :- ::lib.schema/query- previous-stage-columns :- [:sequential ::lib.schema.metadata/column]+ previous-stage-columns :- [:maybe ::lib.metadata.calculation/returned-columns]id-or-name :- [:or ::lib.schema.id/field :string]](log/tracef "Previous stage columns: %s" (pr-str (map (juxt :id :lib/desired-column-alias) previous-stage-columns)))- (when-some [col (resolve-in-previous-stage-metadata-without-updating-keys query previous-stage-columns id-or-name)]+ (when-some [col (resolve-in-previous-stage-returned-columns-without-updating-keys query previous-stage-columns id-or-name)](lib.field.util/update-keys-for-col-from-previous-stage col)))(mu/defn- resolve-in-join :- [:maybe ::lib.metadata.calculation/visible-column]@@ -308,14 +309,18 @@(pr-str id-or-name)(pr-str join-alias)(pr-str stage-number))- (let [join-cols (cond->> (lib.join/join-returned-columns-relative-to-parent-stage query stage-number join nil)+ (let [join-cols (cond->> (lib.metadata.calculation/returned-columns query stage-number join)source-field (remove (fn [col](when-some [col-source-field ((some-fn :fk-field-id :lib/original-fk-field-id) col)](not= col-source-field source-field)))))]- (when-some [col (resolve-in-previous-stage-metadata-and-update-keys query join-cols id-or-name)]- (-> col- (as-> $col (lib.join/column-from-join query stage-number $col join-alias))- (merge (select-keys join [:fk-field-id]))))))+ (or (when-some [col (resolve-in-previous-stage-returned-columns-and-update-keys query join-cols id-or-name)]+ (log/debugf "Got: %s" (pr-str col))+ (-> col+ (as-> $col (lib.join/column-from-join query stage-number $col join-alias))+ (merge (select-keys join [:fk-field-id]))))+ (do+ (log/debugf "Failed to resolve %s in join" (pr-str id-or-name))+ nil))));; a join with this alias does not exist at this stage of the query... try looking recursively in previous;; stage(s)(do@@ -332,13 +337,13 @@;; `Join__COLUMN`... if this fails then fall back to looking for matches that ignore join alias entirely;; e.g. just `COLUMN`(or (when (string? id-or-name)- (resolve-in-previous-stage-metadata-and-update-keys query previous-stage-cols (str join-alias "__" id-or-name)))- (resolve-in-previous-stage-metadata-and-update-keys query previous-stage-cols id-or-name)))+ (resolve-in-previous-stage-returned-columns-and-update-keys query previous-stage-cols (str join-alias "__" id-or-name)))+ (resolve-in-previous-stage-returned-columns-and-update-keys query previous-stage-cols id-or-name)))(do(log/debug "Unable to resolve in previous stage =(")nil))))))-(mu/defn- resolve-in-implicit-join-previous-stage+(mu/defn- resolve-in-implicit-join-previous-stage :- [:maybe ::lib.metadata.calculation/visible-column]"First, try to resolve the implicit join from the previous stage columns -- the join might have already beenperformed there and `:source-field` was specified incorrectly. (You're only supposed to specify this in the stagethe implicit join happens; after that you should drop it and use field name refs instead e.g.@@ -356,7 +361,7 @@(let [previous-stage-cols (filter #(= ((some-fn :fk-field-id :lib/original-fk-field-id) %)source-field-id)(lib.metadata.calculation/returned-columns query previous-stage-number))]- (resolve-in-previous-stage-metadata-and-update-keys query previous-stage-cols id-or-name))))+ (resolve-in-previous-stage-returned-columns-and-update-keys query previous-stage-cols id-or-name))))(mu/defn- find-reified-implicit-join-with-fk-field-id :- [:maybe [:tuple ::lib.schema.join/join :int]]"Find the reified implicit join (i.e., a join added by@@ -394,7 +399,7 @@;; of think of a table as being the Ur-source of the entire query... either way this;; actually still works since `returned-columns` for a table includes desired column;; aliases.- (resolve-in-previous-stage-metadata-without-updating-keys+ (resolve-in-previous-stage-returned-columns-without-updating-keysquery(lib.metadata.calculation/returned-columns query target-table)field-name))))))@@ -459,7 +464,7 @@id-or-name :- ::id-or-name](log/debugf "Resolving %s in previous stage returned columns" (pr-str id-or-name))(when-some [previous-stage-columns (lib.metadata.calculation/returned-columns query previous-stage-number)]- (resolve-in-previous-stage-metadata-and-update-keys query previous-stage-columns id-or-name)))+ (resolve-in-previous-stage-returned-columns-and-update-keys query previous-stage-columns id-or-name)))(mu/defn- resolve-in-card-returned-columns :- [:maybe ::lib.metadata.calculation/visible-column][query :- ::lib.schema/query@@ -468,7 +473,7 @@(log/debugf "Resolving %s in source Card %s metadata" (pr-str id-or-name) (pr-str source-card-id))(when-some [card (lib.metadata/card query source-card-id)](let [card-metadata-columns (lib.metadata.calculation/returned-columns query card)]- (when-some [col (resolve-in-previous-stage-metadata-and-update-keys query card-metadata-columns id-or-name)]+ (when-some [col (resolve-in-previous-stage-returned-columns-and-update-keys query card-metadata-columns id-or-name)](-> col(assoc :lib/source :source/card, :lib/card-id source-card-id))))))@@ -478,15 +483,24 @@id-or-name :- ::id-or-name](log/debugf "Resolving %s in current stage metadata" (pr-str id-or-name))(let [stage (lib.util/query-stage query stage-number)]- (if-some [current-stage-metadata-columns (not-empty (get-in stage [:lib/stage-metadata :columns]))]- ;; TODO (Cam 8/7/25) -- sort of a hacc to add deduplicated names here just so we can properly- ;; use [[resolve-in-previous-stage-metadata-without-updating-keys]], but we don't have any sort of- ;; `resolve-in-current-stage-metadata` function now. I guess it works tho+ (when-some [current-stage-metadata-columns (or (not-empty (get-in stage [:lib/stage-metadata :columns]))+ (do+ (log/debug "stage has no attached metadata")+ nil))](let [current-stage-metadata-columns (lib.field.util/add-deduplicated-names current-stage-metadata-columns)]- (resolve-in-previous-stage-metadata-without-updating-keys query current-stage-metadata-columns id-or-name))- (do- (log/debug "stage has no attached metadata")- nil))))+ (u/prog1 (if (string? id-or-name)+ (some (fn [k]+ (m/find-first #(= (k %) id-or-name)+ current-stage-metadata-columns))+ [:lib/source-column-alias+ :lib/deduplicated-name])+ (m/find-first #(= (:id %) id-or-name) current-stage-metadata-columns))+ (if <>+ (log/debugf "Found match: %s" (pr-str (select-keys <> [:id :lib/source-column-alias :lib/deduplicated-name])))+ (log/debugf "Failed to find match for %s. Found:\n%s"+ (pr-str id-or-name)+ (u/pprint-to-str (map #(select-keys % [:id :lib/source-column-alias :lib/deduplicated-name])+ current-stage-metadata-columns)))))))))(mu/defn- resolve-in-source-card-metadata :- [:maybe ::lib.metadata.calculation/visible-column][query :- ::lib.schema/querydiff --git a/src/metabase/lib/schema/join.cljc b/src/metabase/lib/schema/join.cljcindex c3b57b34e24..aff32f30fa4 100644--- a/src/metabase/lib/schema/join.cljc+++ b/src/metabase/lib/schema/join.cljc@@ -113,9 +113,6 @@[:map{:default {}, :decode/normalize normalize-join}[:lib/type [:= {:default :mbql/join, :decode/normalize common/normalize-keyword} :mbql/join]]- ;; TODO (Cam 7/23/25) -- why would a join need an options map? If we need to add extra keys we can just add them- ;; to the join itself.- [:lib/options ::common/options][:stages [:ref :metabase.lib.schema/stages]][:conditions ::conditions][:alias ::alias]diff --git a/src/metabase/lib/schema/util.cljc b/src/metabase/lib/schema/util.cljcindex 4c8ae6dbf87..e26170c806d 100644--- a/src/metabase/lib/schema/util.cljc+++ b/src/metabase/lib/schema/util.cljc@@ -73,9 +73,11 @@(defn- opts-distinct-key [opts];; Using reduce-kv to remove namespaced keys and some other keys to perform the comparison. This is allegedly faster.- (reduce-kv (fn [acc k _v]+ (reduce-kv (fn [acc k v](if (or (qualified-keyword? k)- (#{:base-type :effective-type} k))+ (#{:base-type :effective-type} k)+ (and (#{:temporal-unit :inherited-temporal-unit} k)+ (= v :default)))(dissoc acc k)acc))optsdiff --git a/src/metabase/lib/util/match.clj b/src/metabase/lib/util/match.cljindex 74e2e7e6080..9883f5c542d 100644--- a/src/metabase/lib/util/match.clj+++ b/src/metabase/lib/util/match.clj@@ -383,6 +383,9 @@(defmacro match-lite"Pattern matching macro, simplified version of [[clojure.core.match]].+ TODO (Cam 9/16/25) -- what exactly is the difference between this and [[match]]? It doesn't recurse? Someone please+ write an explanation here.+Usage:(match-lite valuepattern1 result1diff --git a/src/metabase/lib/walk/util.cljc b/src/metabase/lib/walk/util.cljcindex 2617662b0df..6fbe3cd229a 100644--- a/src/metabase/lib/walk/util.cljc+++ b/src/metabase/lib/walk/util.cljc@@ -4,6 +4,7 @@[metabase.lib.schema :as lib.schema][metabase.lib.schema.id :as lib.schema.id][metabase.lib.schema.template-tag :as lib.schema.template-tag]+ [metabase.lib.util.match :as lib.util.match][metabase.lib.walk :as lib.walk][metabase.util.malli :as mu]))@@ -47,3 +48,29 @@(mapcat :template-tags)(map (fn [[template-tag-name template-tag]](assoc template-tag :lib.walk/template-tag-name template-tag-name))))))++(mu/defn any-native-stage?+ "Returns true if any stage of this query is native."+ [query :- ::lib.schema/query]+ (let [has-native-stage? (volatile! false)]+ (lib.walk/walk-stages+ query+ (fn [_query _path stage]+ (when (and (not @has-native-stage?)+ (= (:lib/type stage) :mbql.stage/native))+ (vreset! has-native-stage? true))+ nil))+ @has-native-stage?))++(mu/defn all-field-ids :- [:set ::lib.schema.id/field]+ "Set of all Field IDs referenced in `:field` refs in `query`."+ [query :- ::lib.schema/query]+ (let [field-ids (volatile! (transient #{}))]+ (lib.walk/walk-clauses+ query+ (fn [_query _path-type _stage-or-join-path clause]+ (lib.util.match/match-lite clause+ [:field _opts (id :guard pos-int?)]+ (vswap! field-ids conj! id))+ nil))+ (persistent! @field-ids)))diff --git a/src/metabase/query_processor/compile.clj b/src/metabase/query_processor/compile.cljindex 42bb8dd0b78..7c69aec1c1b 100644--- a/src/metabase/query_processor/compile.clj+++ b/src/metabase/query_processor/compile.clj@@ -5,7 +5,6 @@[metabase.driver :as driver][metabase.lib.core :as lib][metabase.lib.schema :as lib.schema]- [metabase.lib.walk :as lib.walk][metabase.query-processor.debug :as qp.debug][metabase.query-processor.error-type :as qp.error-type][metabase.query-processor.preprocess :as qp.preprocess]@@ -46,19 +45,6 @@[query](= (:lib/type (lib/query-stage query -1)) :mbql.stage/native))-;;; TODO (Cam 9/11/25) -- move this to `lib.walk.util` once my PR that adds it finally gets reviewed-(mu/defn- any-native-stage?- "Returns true if any stage of this query is native."- [query :- ::lib.schema/query]- (let [has-native-stage? (atom false)]- (lib.walk/walk-stages- query- (fn [_query _path stage]- (when (= (:lib/type stage) :mbql.stage/native)- (reset! has-native-stage? true))- nil))- @has-native-stage?))-(mu/defn- compile* :- ::compiled[query :- ::lib.schema/query](assert (not (:qp/compiled query)) "This query has already been compiled!")@@ -95,7 +81,7 @@(assoc :qp/compiled (compile-preprocessed preprocessed));; if this query is pure-MBQL then we can reliably (re)compile it with inline parameters. If it has any native;; stage then it might already have parameters, and we're not about to try to splice them back in.- (cond-> (not (any-native-stage? preprocessed))+ (cond-> (not (lib/any-native-stage? preprocessed))(assoc :qp/compiled-inline (binding [driver/*compile-with-inline-parameters* true](compile-preprocessed preprocessed)))))))diff --git a/src/metabase/query_processor/middleware/add_remaps.clj b/src/metabase/query_processor/middleware/add_remaps.cljindex 65fe9bd068a..5dc44874e1b 100644--- a/src/metabase/query_processor/middleware/add_remaps.clj+++ b/src/metabase/query_processor/middleware/add_remaps.clj@@ -60,6 +60,7 @@::lib.schema.id/field:string]])+;;; TODO (Cam 9/16/25) -- use [[metabase.lib.schema.util/mbql-clause-distinct-key]] for this(mu/defn- simplify-ref-options :- ::simplified-ref[a-ref :- ::lib.schema.ref/ref](lib/update-options a-ref (fn [opts]@@ -309,16 +310,7 @@(update :new-field-clause lib/with-join-alias (:alias join))))new-fields (into[]- ;; TODO (Cam 7/25/25) the join fields may already include a remap, but `:source-field` or- ;; other distinguishing information doesn't get propagated in refs beyond the stage where the- ;; implicit join happens; thus we should ignore any duplicates with the same `:source-field`.- ;; Joins with multiple remaps to the same table still work because we switch to using name- ;; refs e.g. `NAME` and `NAME_2` instead of duplicate ID refs in this situation --- ;; see [[multiple-fk-remaps-test-in-joins-e2e-test]].- (m/distinct-by (fn [field-ref]- (-> field-ref- simplify-ref-options- (lib/update-options dissoc :source-field))))+ (m/distinct-by simplify-ref-options)(add-fk-remaps-to-fields infos fields))](assoc join :fields new-fields));; there are no remaps to add, discard any changes that happen inside of the join (such as adding additionaldiff --git a/src/metabase/query_processor/middleware/add_source_metadata.clj b/src/metabase/query_processor/middleware/add_source_metadata.cljdeleted file mode 100644index 7741ab5bcfc..00000000000--- a/src/metabase/query_processor/middleware/add_source_metadata.clj+++ /dev/null@@ -1,136 +0,0 @@-(ns metabase.query-processor.middleware.add-source-metadata- (:require- [clojure.walk :as walk]- ;; legacy usage -- don't use Legacy MBQL utils in QP code going forward, prefer Lib. This will be updated to use- ;; Lib soon- ^{:clj-kondo/ignore [:discouraged-namespace]}- [metabase.legacy-mbql.schema :as mbql.s]- ^{:clj-kondo/ignore [:discouraged-namespace]}- [metabase.legacy-mbql.util :as mbql.u]- [metabase.lib.metadata :as lib.metadata]- [metabase.lib.util.match :as lib.util.match]- [metabase.query-processor.interface :as qp.i]- [metabase.query-processor.store :as qp.store]- [metabase.request.core :as request]- [metabase.util.log :as log]- [metabase.util.malli :as mu]))--(defn- has-same-fields-as-nested-source?- "Whether this source query itself has a nested source query, and will have the exact same fields in the results as its- nested source. If this is the case, we can return the `source-metadata` for the nested source as-is, if it is- present."- [{nested-source-query :source-query- nested-source-metadata :source-metadata- breakouts :breakout- aggregations :aggregation- fields :fields}]- (when nested-source-query- (and (every? empty? [breakouts aggregations])- (or (empty? fields)- (and (= (count fields) (count nested-source-metadata))- (every? #(lib.util.match/match-one % [:field (_ :guard string?) _])- fields))))))--(mu/defn- native-source-query->metadata :- [:maybe [:sequential ::mbql.s/legacy-column-metadata]]- "Given a `source-query`, return the source metadata that should be added at the parent level (i.e., at the same- level where this `source-query` was present.) This metadata is used by other middleware to determine what Fields to- expect from the source query."- [{nested-source-metadata :source-metadata, :as source-query} :- mbql.s/SourceQuery]- ;; If the source query has a nested source with metadata and does not change the fields that come back, return- ;; metadata as-is- (if (has-same-fields-as-nested-source? source-query)- nested-source-metadata- ;; Otherwise we cannot determine the metadata automatically; usually, this is because the source query itself has- ;; a native source query- (do- (when-not qp.i/*disable-qp-logging*- (log/warn "Cannot infer `:source-metadata` for source query with native source query without source metadata."- {:source-query source-query}))- nil)))--(mu/defn mbql-source-query->metadata :- [:maybe [:sequential ::mbql.s/legacy-column-metadata]]- "Preprocess a `source-query` so we can determine the result columns."- [source-query :- mbql.s/MBQLQuery]- (try- (let [cols (request/as-admin- ((requiring-resolve 'metabase.query-processor.preprocess/query->expected-cols)- {:database (:id (lib.metadata/database (qp.store/metadata-provider)))- :type :query- ;; don't add remapped columns to the source metadata for the source query, otherwise we're going- ;; to end up adding it again when the middleware runs at the top level- :query (assoc-in source-query [:middleware :disable-remaps?] true)}))]- (remove :remapped_from cols))- (catch Throwable e- (log/errorf e "Error determining expected columns for query: %s" (ex-message e))- nil)))--(mu/defn- add-source-metadata :- [:map- [:source-metadata- {:optional true}- [:maybe [:sequential ::mbql.s/legacy-column-metadata]]]]- [{{native-source-query? :native, :as source-query} :source-query, :as inner-query} :- :map]- (let [metadata ((if native-source-query?- native-source-query->metadata- mbql-source-query->metadata) source-query)- metadata (when metadata- (let [unique-name-fn (mbql.u/unique-name-generator)]- (for [col metadata- :let [original-name ((some-fn :lib/original-name :name) col)]]- (assoc col- :lib/original-name original-name- :lib/deduplicated-name (unique-name-fn original-name)))))]- (cond-> inner-query- (seq metadata) (assoc :source-metadata metadata))))--(defn- legacy-source-metadata-without-field-ref?- "Whether this source metadata is *legacy* source metadata from < 0.38.0. Legacy source metadata did not include- `:field_ref` or `:id`, which made it hard to correctly construct queries with. For MBQL queries, we're better off- ignoring legacy source metadata and using [[metabase.query-processor.preprocess/query->expected-cols]] to infer the- source metadata rather than relying on old stuff that can produce incorrect queries. See #14788 for more- information."- [source-metadata]- (and (seq source-metadata)- (every? nil? (map :field_ref source-metadata))))--(defn- should-add-source-metadata?- "Should we add `:source-metadata` about the `:source-query` in this map? True if all of the following are true:-- * The map (e.g. an 'inner' MBQL query or a Join) has a `:source-query`-- * The map does not *already* have `:source-metadata`, or the `:source-metadata` is 'legacy' source metadata from- versions < 0.38.0-- * The `:source-query` is an MBQL query, or a native source query with `:source-metadata`"- [{{native-source-query? :native- source-query-has-source-metadata? :source-metadata- :as source-query} :source-query- :keys [source-metadata]}]- (let [source-metadata source-metadata]- (and source-query- (let [valid-source-metadata? (and source-metadata- (not (legacy-source-metadata-without-field-ref? source-metadata)))]- (not valid-source-metadata?))- (or (not native-source-query?)- source-query-has-source-metadata?- (:qp/stage-is-from-source-card source-query)))))--(defn- maybe-add-source-metadata [x]- (if (and (map? x) (should-add-source-metadata? x))- (add-source-metadata x)- x))--(defn- add-source-metadata-at-all-levels [inner-query]- (walk/postwalk maybe-add-source-metadata inner-query))--(mu/defn add-source-metadata-for-source-queries :- ::mbql.s/Query- "Middleware that attempts to recursively add `:source-metadata`, if not already present, to any maps with a- `:source-query`.-- `:source-metadata` is information about the columns we can expect to come back from the source- query; this is added automatically for source queries added via the `card__id` source table form, but for *explicit*- source queries that do not specify this information, we can often infer it by looking at the shape of the source- query."- [{query-type :type, :as query} :- ::mbql.s/Query]- (if-not (= query-type :query)- query- (update query :query add-source-metadata-at-all-levels)))diff --git a/src/metabase/query_processor/middleware/ensure_joins_use_source_query.clj b/src/metabase/query_processor/middleware/ensure_joins_use_source_query.cljdeleted file mode 100644index 1dc9b9b62e8..00000000000--- a/src/metabase/query_processor/middleware/ensure_joins_use_source_query.clj+++ /dev/null@@ -1,30 +0,0 @@-(ns metabase.query-processor.middleware.ensure-joins-use-source-query- (:require- [metabase.lib.schema :as lib.schema]- [metabase.lib.walk :as lib.walk]- [metabase.util.malli :as mu]))--(mu/defn ensure-joins-use-source-query :- ::lib.schema/query- "Super simple middleware to ensure that joins keep-- {:source-query {:source-table ...}}-- instead of getting automatically collapsed back to-- {:source-table ...}-- when the query is converted to legacy. The main reason we want to do this is- so [[metabase.query-processor.middleware.add-implicit-clauses/add-implicit-clauses]] will add `:fields` to the- join's `:source-query` so that [[metabase.query-processor.middleware.add-remaps/add-remapped-columns]] can add- remaps as appropriate.-- This is accomplished by adding a dummy key to the last stage of the join; [[metabase.lib.convert/->legacy-MBQL]] will- not collapse the last stage into the top level of the join if extra keys are present.-- Once we convert the `add-implicit-clauses` middleware to use Lib we can remove this middleware entirely."- [query :- ::lib.schema/query]- (lib.walk/walk- query- (fn [_query path-type _path join]- (when (= path-type :lib.walk/join)- (assoc-in join [:stages (dec (count (:stages join))) ::dummy-key] true)))))diff --git a/src/metabase/query_processor/middleware/resolve_fields.clj b/src/metabase/query_processor/middleware/resolve_fields.cljindex 25470be9c30..8629f5762f3 100644--- a/src/metabase/query_processor/middleware/resolve_fields.clj+++ b/src/metabase/query_processor/middleware/resolve_fields.clj@@ -1,14 +1,11 @@(ns metabase.query-processor.middleware.resolve-fields"Middleware that resolves the Fields referenced by a query."(:require+ [metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]- [metabase.lib.schema :as lib.schema]- [metabase.lib.schema.id :as lib.schema.id]- [metabase.lib.util.match :as lib.util.match][metabase.query-processor.error-type :as qp.error-type][metabase.util :as u]- [metabase.util.i18n :refer [tru]]- [metabase.util.malli :as mu]))+ [metabase.util.i18n :refer [tru]]))(defn- resolve-fields-with-ids![metadata-providerable field-ids]@@ -21,23 +18,12 @@field-ids))](recur metadata-providerable parent-ids)))-(mu/defn- field-ids :- [:set ::lib.schema.id/field]- [query :- ::lib.schema/query]- (into #{}- (lib.util.match/match (:stages query)- [:field _opts (id :guard pos-int?)]- id-- ;; stage metadata- {:lib/type :metadata/column, :id (id :guard pos-int?)}- id)))-;;; TODO (Cam 9/10/25) -- give this a more accurate name like `prefetch-all-fields` or;;; `warm-metadata-provider-cache-with-all-fields` or something.(defn resolve-fields"Resolve all field referenced in the `query`, and store them in the Metadata Provider."[query]- (let [ids (field-ids query)]+ (let [ids (lib/all-field-ids query)](try(u/prog1 query(resolve-fields-with-ids! query ids))diff --git a/src/metabase/query_processor/middleware/resolve_joins.clj b/src/metabase/query_processor/middleware/resolve_joins.cljindex 38e9d53a3e5..651c60849aa 100644--- a/src/metabase/query_processor/middleware/resolve_joins.clj+++ b/src/metabase/query_processor/middleware/resolve_joins.clj@@ -3,148 +3,81 @@options, and adds information to the query about what joins should be done and how they should be performed."(:refer-clojure :exclude [alias])(:require+ [clojure.string :as str][medley.core :as m]- ;; legacy usage -- don't use Legacy MBQL utils in QP code going forward, prefer Lib. This will be updated to use- ;; Lib soon- ^{:clj-kondo/ignore [:discouraged-namespace]}- [metabase.legacy-mbql.schema :as mbql.s]- [metabase.lib.metadata :as lib.metadata]- [metabase.lib.schema.id :as lib.schema.id]- [metabase.lib.util :as lib.util]- [metabase.lib.util.match :as lib.util.match]- [metabase.query-processor.error-type :as qp.error-type]- [metabase.query-processor.store :as qp.store]- [metabase.util :as u]- [metabase.util.i18n :refer [tru]]+ [metabase.lib.core :as lib]+ [metabase.lib.schema :as lib.schema]+ [metabase.lib.schema.join :as lib.schema.join]+ [metabase.lib.schema.util :as lib.schema.util]+ [metabase.lib.walk :as lib.walk][metabase.util.malli :as mu]))-(def ^:private Joins- "Schema for a non-empty sequence of Joins. Unlike [[mbql.s/Joins]], this does not enforce the constraint that all join- aliases be unique."- [:sequential {:min 1} mbql.s/Join])--(def ^:private UnresolvedMBQLQuery- "Schema for the parts of the query we're modifying. For use in the various intermediate transformations in the- middleware."- [:map- [:joins [:sequential mbql.s/Join]]- [:fields {:optional true} mbql.s/Fields]])--(def ^:private ResolvedMBQLQuery- "Schema for the final results of this middleware."- [:and- UnresolvedMBQLQuery- [:fn- {:error/message "Valid MBQL query where `:joins` `:fields` is sequence of Fields or removed"}- (fn [{:keys [joins]}]- (every?- (fn [{:keys [fields]}]- (or- (empty? fields)- (sequential? fields)))- joins))]])--;;; +----------------------------------------------------------------------------------------------------------------+-;;; | Resolving Tables & Fields / Saving in QP Store |-;;; +----------------------------------------------------------------------------------------------------------------+--(mu/defn- resolve-fields! :- :nil- [joins :- Joins]- (lib.metadata/bulk-metadata-or-throw (qp.store/metadata-provider)- :metadata/column- (lib.util.match/match joins [:field (id :guard integer?) _] id))- nil)--(mu/defn- resolve-tables! :- :nil- "Add Tables referenced by `:joins` to the Query Processor Store. This is only really needed for implicit joins,- because their Table references are added after `resolve-source-tables` runs."- [joins :- Joins]- (lib.metadata/bulk-metadata-or-throw (qp.store/metadata-provider)- :metadata/table- (remove nil? (map :source-table joins)))- nil)--;;; +----------------------------------------------------------------------------------------------------------------+-;;; | :Joins Transformations |-;;; +----------------------------------------------------------------------------------------------------------------+--(def ^:private default-join-alias "__join")--(mu/defn- merge-defaults :- mbql.s/Join+(mu/defn- merge-defaults :- ::lib.schema.join/join[join]- (merge {:alias default-join-alias, :strategy :left-join} join))--(defn- source-metadata->fields [{:keys [alias], :as join} source-metadata]- (when-not (seq source-metadata)- (throw (ex-info (tru "Cannot use :fields :all in join against source query unless it has :source-metadata.")- {:join join})))+ ;;; TODO (Cam 9/16/25) -- if we made `:strategy` a required key with a `:default` value in+ ;;; `:metabase.lib.schema.join/join` then Lib normalization could handle this for us and we wouldn't need QP+ ;;; middleware to do it. Seems like the better way to deal with defaults. (QUE-2469)+ (merge {:strategy :left-join}+ (when (str/starts-with? (:alias join) lib/legacy-default-join-alias)+ {:qp/keep-default-join-alias true})+ join))++(defn- join-field-refs [cols](let [duplicate-ids (into #{}(keep (fn [[item freq]](when (> freq 1)item)))- (frequencies (map :id source-metadata)))]- (for [{field-name :name- base-type :base_type- field-id :id- field-ref :field_ref} source-metadata]- ;; If `field-ref` is an id-based reference, only use it if the source query uses it.- (or (when (and field-id (not (contains? duplicate-ids field-id)))- (lib.util.match/match-one field-ref- [:field (id :guard pos-int?) _opts]- [:field field-id {:join-alias alias}]))- [:field field-name {:base-type base-type, :join-alias alias}]))))--(mu/defn- sorted-implicit-fields-for-table :- mbql.s/Fields- "For use when adding implicit Field IDs to a query. Return a sequence of field clauses, sorted by the rules listed- in [[metabase.query-processor.sort]], for all the Fields in a given Table."- [table-id :- ::lib.schema.id/table]- (let [fields (lib.metadata/active-fields (qp.store/metadata-provider) table-id)]- (when (empty? fields)- (throw (ex-info (tru "No fields found for table {0}." (pr-str (:name (lib.metadata/table (qp.store/metadata-provider) table-id))))- {:table-id table-id- :type qp.error-type/invalid-query})))- (mapv- (fn [field]- [:field (u/the-id field) nil])- fields)))--(mu/defn- handle-all-fields :- mbql.s/Join+ (frequencies (map :id cols)))]+ ;; TODO (Cam 9/16/25) -- forcing refs of certain types like this is wonky, but when I try to change this tons of+ ;; stuff breaks. Forcing ID refs doesn't work because a join can return multiple versions of the same column+ ;; bucketed in different ways in previous stages; joins thus ought to be using field name refs; but this ends up+ ;; breaking a ton of stuff, especially `lib.equality`... #63109 was my attempt to make this stuff work when using+ ;; field name refs for joins but it's a long way off from landing.+ ;;+ ;; NOCOMMIT+ (for [{field-id :id, :as col} cols+ :let [[_tag opts id-or-name, :as field-ref] (lib/ref col)+ force-id-ref? (and (string? id-or-name)+ field-id+ (not (contains? duplicate-ids field-id)))+ force-field-name-ref? (and (pos-int? id-or-name)+ (contains? duplicate-ids field-id))]]++ (cond+ force-id-ref?+ [:field opts field-id]++ force-field-name-ref?+ [:field opts (:lib/source-column-alias col)]++ :else+ field-ref))))++(mu/defn- handle-all-fields :- ::lib.schema.join/join"Replace `:fields :all` in a join with an appropriate list of Fields."- [{:keys [source-table source-query alias fields source-metadata], :as join} :- mbql.s/Join]+ [query :- ::lib.schema/query+ path :- ::lib.walk/path+ {:keys [fields], :as join} :- ::lib.schema.join/join](mergejoin(when (= fields :all)- {:fields (if source-query- (source-metadata->fields join source-metadata)- (for [[_ id-or-name opts] (sorted-implicit-fields-for-table source-table)]- [:field id-or-name (assoc opts :join-alias alias)]))})))--(defn- deduplicate-aliases []- (let [unique-name-generator (lib.util/non-truncating-unique-name-generator)]- (map (fn [join]- (update join :alias unique-name-generator)))))--(mu/defn- resolve-references :- Joins- [joins :- Joins]- (resolve-tables! joins)- (u/prog1 (into []- (comp (map merge-defaults)- (map handle-all-fields)- (deduplicate-aliases))- joins)- (resolve-fields! <>)))--(declare resolve-joins-in-mbql-query-all-levels)--(mu/defn- resolve-join-source-queries :- Joins- [joins :- Joins]- (for [{:keys [source-query], :as join} joins]- (cond-> join- source-query resolve-joins-in-mbql-query-all-levels)))--;;; +----------------------------------------------------------------------------------------------------------------+-;;; | MBQL-Query Transformations |-;;; +----------------------------------------------------------------------------------------------------------------++ ;; do not `:include-remaps?` here, they will get added by the [[metabase.query-processor.middleware.add-remaps]]+ ;; middleware.+ {:fields (join-field-refs+ (lib.walk/apply-f-for-stage-at-path+ lib/join-fields-to-add-to-parent-stage+ query+ path+ join+ {:include-remaps? false}))})))++(mu/defn- resolve-join :- ::lib.schema.join/join+ [query :- ::lib.schema/query+ path :- ::lib.walk/path+ join :- ::lib.schema.join/join]+ (->> join+ merge-defaults+ (handle-all-fields query path)))(defn- joins->fields"Return a flattened list of all `:fields` referenced in `joins`."@@ -155,89 +88,64 @@;; make sure the field ref has `:join-alias`... it already SHOULD but if the query is NAUGHTY then;; we better just add it in to be safe. In #61398 which is pending I actually make this happen;; automatically in MBQL 5 normalization, so we can take this out eventually.- (for [[tag id-or-name opts] fields]- [tag id-or-name (assoc opts- :join-alias (:alias join)- ;; Any coercion or temporal bucketing will already have been done in the- ;; subquery for the join itself. Mark the parent ref to make sure it is- ;; not double-coerced, which leads to SQL errors.- :qp/ignore-coercion true)]))))+ (for [field-ref fields]+ (-> field-ref+ (lib/with-join-alias (:alias join))+ ;; Any coercion or temporal bucketing will already have been done in the+ ;; subquery for the join itself. Mark the parent ref to make sure it is+ ;; not double-coerced, which leads to SQL errors.+ (lib/update-options assoc :qp/ignore-coercion true))))))joins))(defn- should-add-join-fields?"Should we append the `:fields` from `:joins` to the parent-level query's `:fields`? True unless the parent-levelquery has breakouts or aggregations."- [{breakouts :breakout, aggregations :aggregation}]+ [{breakouts :breakout, aggregations :aggregation, :as _stage}](every? empty? [aggregations breakouts]))+;;; TODO (Cam 9/16/25) -- update this to use [[metabase.lib.stage/add-cols-from-join]] or share more logic with it(defn- append-join-fields- "This (supposedly) matches the behavior of [[metabase.lib.stage/add-cols-from-join]]. When we migrate this namespace- to Lib we can maybe use that."+ "This (supposedly) matches the behavior of [[metabase.lib.stage/add-cols-from-join]]."[fields join-fields];; we shouldn't consider different type info to mean two Fields are different even if everything else is the same. So;; give everything `:base-type` of `:type/*` (it will complain if we remove `:base-type` entirely from fields with a;; string name)- (letfn [(opts-signature [opts]- (not-empty- (merge- (u/select-non-nil-keys opts [:join-alias :binning])- ;; for purposes of deduplicating stuff, temporal unit = default is the same as not specifying temporal- ;; unit at all. Should that be part of normalization? Maybe, but there is some logic around adding default- ;; temporal bucketing that we don't do if `:default` is explicitly specified.- (when-let [temporal-unit (:temporal-unit opts)]- (when-not (= temporal-unit :default)- {:temporal-unit temporal-unit})))))- (ref-signature [[tag id-or-name opts, :as _ref]]- [tag id-or-name (opts-signature opts)])]- (into []- (comp cat- (m/distinct-by ref-signature))- [fields join-fields])))+ (into []+ (comp cat+ (m/distinct-by lib.schema.util/mbql-clause-distinct-key))+ [fields (lib/fresh-uuids join-fields)]))(defn append-join-fields-to-fields"Add the fields from join `:fields`, if any, to the parent-level `:fields`."- [inner-query join-fields]- (cond-> inner-query+ [stage join-fields]+ (cond-> stage(seq join-fields) (update :fields append-join-fields join-fields)))-(mu/defn- merge-joins-fields :- UnresolvedMBQLQuery+(mu/defn- merge-joins-fields :- ::lib.schema/stage.mbql"Append the `:fields` from `:joins` into their parent level as appropriate so joined columns appear in the finalquery results, and remove the `:fields` entry for all joins.If the parent-level query has breakouts and/or aggregations, this function won't append the joins fields to theparent level, because we should only be returning the ones from the ags and breakouts in the final results."- [{:keys [joins], :as inner-query} :- UnresolvedMBQLQuery]- (let [join-fields (when (should-add-join-fields? inner-query)+ [{:keys [joins], :as stage} :- ::lib.schema/stage.mbql]+ (let [join-fields (when (should-add-join-fields? stage)(joins->fields joins))- ;; remove remaining keyword `:fields` like `:none` from joins- inner-query (update inner-query :joins (fn [joins]- (mapv (fn [{:keys [fields], :as join}]- (cond-> join- (keyword? fields) (dissoc :fields)))- joins)))]- (append-join-fields-to-fields inner-query join-fields)))--(mu/defn- resolve-joins-in-mbql-query :- ResolvedMBQLQuery- [query :- mbql.s/MBQLQuery]+ stage (update stage :joins (fn [joins]+ (mapv (fn [{:keys [fields], :as join}]+ (cond-> join+ (keyword? fields) (dissoc :fields)))+ joins)))]+ (append-join-fields-to-fields stage join-fields)))++(mu/defn resolve-joins :- ::lib.schema/query+ "1. Walk joins and merge defaults like `:strategy :left-join`+ 2. Walk joins and resolve `:fields :all` to a vector of field refs+ 3. Walk stages and merge in `:fields` from `:joins`"+ [query :- ::lib.schema/query](-> query- (update :joins (comp resolve-join-source-queries resolve-references))- merge-joins-fields))--;;; +----------------------------------------------------------------------------------------------------------------+-;;; | Middleware & Boring Recursive Application Stuff |-;;; +----------------------------------------------------------------------------------------------------------------+--(defn- resolve-joins-in-mbql-query-all-levels- [{:keys [joins source-query], :as query}]- (cond-> query- (seq joins) resolve-joins-in-mbql-query- source-query (update :source-query resolve-joins-in-mbql-query-all-levels)))--;; TODO (Cam 9/10/25) -- once we convert this to Lib we can remove-;; the [[metabase.query-processor.middleware.ensure-joins-use-source-query/ensure-joins-use-source-query]] middleware-;; entirely-(defn resolve-joins- "Add any Tables and Fields referenced by the `:joins` clause to the QP store."- [{inner-query :query, :as outer-query}]- (cond-> outer-query- inner-query (update :query resolve-joins-in-mbql-query-all-levels)))+ (lib.walk/walk (fn [query path-type path join]+ (when (= path-type :lib.walk/join)+ (resolve-join query path join))))+ (lib.walk/walk-stages (fn [__query _path stage]+ (when (seq (:joins stage))+ (merge-joins-fields stage))))))diff --git a/src/metabase/query_processor/middleware/resolve_source_table.clj b/src/metabase/query_processor/middleware/resolve_source_table.cljindex 906d16c930b..7c3fbdab2ab 100644--- a/src/metabase/query_processor/middleware/resolve_source_table.clj+++ b/src/metabase/query_processor/middleware/resolve_source_table.clj@@ -1,28 +1,13 @@(ns metabase.query-processor.middleware.resolve-source-table"Fetches Tables corresponding to any `:source-table` IDs anywhere in the query."(:require- [metabase.lib.metadata :as lib.metadata]- [metabase.lib.schema.id :as lib.schema.id]- [metabase.lib.walk :as lib.walk]- [metabase.util.malli :as mu]))--;;; TODO (Cam 9/11/25) -- update this to use the `lib.walk.util` functions once #63321 is merged-(mu/defn ^:private query->source-table-ids :- [:maybe [:set {:min 1} ::lib.schema.id/table]]- "Fetch a set of all `:source-table` IDs anywhere in `query`."- [query]- (let [source-table-ids (volatile! #{})]- (lib.walk/walk-stages- query- (fn [_query _path {:keys [source-table], :as _stage}]- (when source-table- (vswap! source-table-ids conj source-table))- nil))- (not-empty @source-table-ids)))+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]))(defn resolve-source-tables"Middleware that will take any `:source-table`s (integer IDs) anywhere in the query and fetch and save thecorresponding Table in the Query Processor Store."[query];; this is done for side effects- (lib.metadata/bulk-metadata-or-throw query :metadata/table (query->source-table-ids query))+ (lib.metadata/bulk-metadata-or-throw query :metadata/table (lib/all-source-table-ids query))query)diff --git a/src/metabase/query_processor/preprocess.clj b/src/metabase/query_processor/preprocess.cljindex 90ea02d8d9f..307123a106f 100644--- a/src/metabase/query_processor/preprocess.clj+++ b/src/metabase/query_processor/preprocess.clj@@ -15,7 +15,6 @@[metabase.query-processor.middleware.add-implicit-clauses :as qp.add-implicit-clauses][metabase.query-processor.middleware.add-implicit-joins :as qp.add-implicit-joins][metabase.query-processor.middleware.add-remaps :as qp.add-remaps]- [metabase.query-processor.middleware.add-source-metadata :as qp.add-source-metadata][metabase.query-processor.middleware.annotate :as annotate][metabase.query-processor.middleware.auto-bucket-datetimes :as qp.auto-bucket-datetimes][metabase.query-processor.middleware.auto-parse-filter-values :as auto-parse-filter-values]@@ -25,7 +24,6 @@[metabase.query-processor.middleware.cumulative-aggregations :as qp.cumulative-aggregations][metabase.query-processor.middleware.desugar :as desugar][metabase.query-processor.middleware.drop-fields-in-summaries :as drop-fields-in-summaries]- [metabase.query-processor.middleware.ensure-joins-use-source-query :as ensure-joins-use-source-query][metabase.query-processor.middleware.enterprise :as qp.middleware.enterprise][metabase.query-processor.middleware.expand-aggregations :as expand-aggregations][metabase.query-processor.middleware.expand-macros :as expand-macros]@@ -110,11 +108,7 @@(ensure-mbql5 #'parameters/substitute-parameters)(ensure-mbql5 #'qp.resolve-source-table/resolve-source-tables)(ensure-mbql5 #'qp.auto-bucket-datetimes/auto-bucket-datetimes)- ;; TODO (Cam 9/10/25) -- once we convert the [[metabase.query-processor.middleware.resolve-joins/resolve-joins]]- ;; middleware to Lib we can remove this middleware entirely- (ensure-mbql5 #'ensure-joins-use-source-query/ensure-joins-use-source-query)(ensure-mbql5 #'reconcile-bucketing/reconcile-breakout-and-order-by-bucketing)- (ensure-legacy #'qp.add-source-metadata/add-source-metadata-for-source-queries)(ensure-mbql5 #'qp.middleware.enterprise/apply-impersonation)(ensure-mbql5 #'qp.middleware.enterprise/attach-destination-db-middleware)(ensure-mbql5 #'qp.middleware.enterprise/apply-sandboxing)@@ -123,14 +117,14 @@;; this needs to be done twice, once before adding remaps (since we want to add remaps inside joins) and then again;; after adding any implicit joins. Implicit joins do not need to get remaps since we only use them for fetching;; specific columns.- (ensure-legacy #'resolve-joins/resolve-joins)+ (ensure-mbql5 #'resolve-joins/resolve-joins)(ensure-mbql5 #'qp.add-remaps/add-remapped-columns)#'qp.resolve-fields/resolve-fields ; this middleware actually works with either MBQL 5 or legacy(ensure-mbql5 #'binning/update-binning-strategy)(ensure-mbql5 #'desugar/desugar)(ensure-mbql5 #'qp.add-default-temporal-unit/add-default-temporal-unit)(ensure-mbql5 #'qp.add-implicit-joins/add-implicit-joins)- (ensure-legacy #'resolve-joins/resolve-joins) ; #61398+ (ensure-mbql5 #'resolve-joins/resolve-joins)(ensure-mbql5 #'fix-bad-field-id-refs/fix-bad-field-id-refs)(ensure-mbql5 #'qp.remove-inactive-field-refs/remove-inactive-field-refs);; yes, this is called a second time, because we need to handle any joins that got addeddiff --git a/test/metabase/lib/field/resolution_test.cljc b/test/metabase/lib/field/resolution_test.cljcindex 337f01e33ef..7a097313216 100644--- a/test/metabase/lib/field/resolution_test.cljc+++ b/test/metabase/lib/field/resolution_test.cljc@@ -34,11 +34,12 @@(doseq [f [#'lib/metadata#'lib.field.resolution/resolve-field-ref]](testing f- (is (=? {:name "sum"- :display-name "sum of User ID"- :base-type :type/Integer- :semantic-type :type/FK- :lib/source :source/native}+ (is (=? {:name "sum"+ :display-name "sum of User ID"+ :base-type :type/Integer+ :semantic-type :type/FK+ :lib/source :source/native+ ::lib.field.resolution/fallback-metadata? (symbol "nil #_\"key is not present.\"")}(lib/metadata(lib.tu/native-query)-1@@ -1447,14 +1448,14 @@{:stages [{:joins [{:alias "J":stages [{:source-table (meta/id :venues):joins [{:alias "CATEGORIES__via__ID"- :fk-field-id (meta/id :venues :category-id)+ :fk-field-id (meta/id :venues :id):stages [{:source-table (meta/id :categories):fields [[:field {} (meta/id :categories :id)][:field {} (meta/id :categories :name)]]}]:conditions [[:= {} 1 1]]:fields :none}{:alias "CATEGORIES__via__CATEGORY_ID"- :fk-field-id (meta/id :venues :id)+ :fk-field-id (meta/id :venues :category-id):stages [{:source-table (meta/id :categories):fields [[:field {} (meta/id :categories :id)][:field {} (meta/id :categories :name)]]}]@@ -1508,3 +1509,66 @@:lib/source-column-alias "CATEGORIES__via__CATEGORY_ID__NAME":metabase.lib.join/join-alias "J"}(lib.field.resolution/resolve-field-ref query 0 field-ref)))))))))++(deftest ^:parallel resolve-implicit-column-test+ (testing "fields implicitly joined in a previous stage"+ (let [query (-> (lib/query+ meta/metadata-provider+ (lib.tu.macros/mbql-query venues+ {:fields [$category-id->categories.name]+ :order-by [[:asc $id]]}))+ lib/append-stage)]+ (is (=? {:id (meta/id :categories :name)+ :table-id (meta/id :categories)+ :lib/original-fk-field-id (meta/id :venues :category-id)+ ::lib.field.resolution/fallback-metadata? (symbol "nil #_\"key is not present.\"")}+ (lib.field.resolution/resolve-field-ref query -1 [:field+ {:base-type :type/Text, :effective-type :type/Text, :lib/uuid "ee91656b-08c7-4ca2-9f9e-5845e2edd80a"}+ "CATEGORIES__via__CATEGORY_ID__NAME"]))+ ":fk-field-id needs to get propagated as :lib/previous-stage-fk-field-id this to work correctly"))))++(deftest ^:parallel resolve-by-name-from-join-test+ (let [query (lib/query+ meta/metadata-provider+ {:lib/type :mbql/query+ :stages [{:lib/type :mbql.stage/mbql+ :source-table (meta/id :people)+ :joins [{:lib/type :mbql/join+ :strategy :left-join+ :alias "Q1"+ :stages [{:lib/type :mbql.stage/mbql+ :source-table (meta/id :orders)+ :fields [[:field {} (meta/id :orders :id)]+ [:field {:base-type :type/Integer, :join-alias "O"} "ID"]]+ :joins [{:lib/type :mbql/join+ :strategy :left-join+ :alias "O"+ :stages [{:lib/type :mbql.stage/mbql+ :source-table (meta/id :orders)+ :fields [[:field {} (meta/id :orders :id)]]}]+ :conditions [[:=+ {}+ [:field {} (meta/id :orders :id)]+ [:field {:join-alias "O"} (meta/id :orders :id)]]]+ :fields [[:field {:base-type :type/Integer, :join-alias "O"} "ID"]]}]}+ {:lib/type :mbql.stage/mbql+ :fields [[:field {:base-type :type/BigInteger} "ID"]+ [:field {:base-type :type/Integer} "O__ID"]]}]+ :conditions [[:= {}+ [:field {} (meta/id :people :id)]+ [:field {:base-type :type/BigInteger, :join-alias "Q1"} (meta/id :orders :user-id)]]]+ :fields [[:field {:base-type :type/Integer, :join-alias "Q1"} "ID"]+ [:field {:base-type :type/Integer, :join-alias "Q1"} "o__ID"]]}]+ :fields [[:field {} (meta/id :orders :id)]+ [:field {:base-type :type/Integer, :join-alias "Q1"} "ID"]+ [:field {:base-type :type/Integer, :join-alias "Q1"} "O__ID"]]}]+ :database (meta/id)})]+ (is (=? {:id (meta/id :orders :id)+ :table-id (meta/id :orders)+ :metabase.lib.join/join-alias "Q1"+ :lib/source-column-alias "O__ID"+ ::lib.field.resolution/fallback-metadata? (symbol "nil #_\"key is not present.\"")}+ (lib.field.resolution/resolve-field-ref query -1 [:field {:base-type :type/Integer+ :lib/uuid "00000000-0000-0000-0000-000000000000"+ :join-alias "Q1"}+ "O__ID"])))))diff --git a/test/metabase/lib/schema/join_test.cljc b/test/metabase/lib/schema/join_test.cljcindex b191cabef6d..3bfd512d6b9 100644--- a/test/metabase/lib/schema/join_test.cljc+++ b/test/metabase/lib/schema/join_test.cljc@@ -20,7 +20,6 @@(is (=? {:lib/type "missing required key":stages [{:joins [{:stages [[{:lib/type "missing required key"}"Invalid stage :lib/type: expected :mbql.stage/native or :mbql.stage/mbql"]]- :lib/options "missing required key":conditions "should have at least 1 elements"}]}]}(mu.humanize/humanize (mr/explain ::lib.schema/query {:stages [{:lib/type :mbql.stage/mbql:joins [{:lib/type :mbql/joindiff --git a/test/metabase/lib/schema/util_test.cljc b/test/metabase/lib/schema/util_test.cljcindex e091ace6028..3f961ba2985 100644--- a/test/metabase/lib/schema/util_test.cljc+++ b/test/metabase/lib/schema/util_test.cljc@@ -77,6 +77,13 @@[:field {:lib/uuid "00000000-0000-0000-0000-000000000001", :effective-type :type/Number} 1]][[:field {:lib/uuid "00000000-0000-0000-0000-000000000000", :effective-type :type/Integer} 1]+ [:field {:lib/uuid "00000000-0000-0000-0000-000000000001"} 1]]++ ;; do not consider (inherited-)temporal-unit = default and no temporal-unit to be distinct+ [[:field {:lib/uuid "00000000-0000-0000-0000-000000000000", :temporal-unit :default} 1]+ [:field {:lib/uuid "00000000-0000-0000-0000-000000000001"} 1]]++ [[:field {:lib/uuid "00000000-0000-0000-0000-000000000000", :inherited-temporal-unit :default} 1][:field {:lib/uuid "00000000-0000-0000-0000-000000000001"} 1]]))(deftest ^:parallel distinct-mbql-clauses-schema-testdiff --git a/test/metabase/lib/walk/util_test.cljc b/test/metabase/lib/walk/util_test.cljcindex af02823e1ad..decfdcd40d6 100644--- a/test/metabase/lib/walk/util_test.cljc+++ b/test/metabase/lib/walk/util_test.cljc@@ -4,7 +4,8 @@[metabase.lib.core :as lib][metabase.lib.test-metadata :as meta][metabase.lib.test-util.macros :as lib.tu.macros]- [metabase.lib.walk.util :as lib.walk.util]))+ [metabase.lib.walk.util :as lib.walk.util]+ [metabase.util.malli :as mu]))(deftest ^:parallel all-source-table-ids-test(testing (str "make sure that `all-table-ids` can properly find all Tables in the query, even in cases where a map "@@ -31,3 +32,15 @@:condition [:=$checkins.user-id&U.users.id]}]}))))))++(deftest ^:parallel all-field-ids-test+ (mu/disable-enforcement+ (is (= #{1 2}+ (lib/all-field-ids+ {:lib/type :mbql/query+ :database 1+ :stages [{:lib/type :mbql.stage/mbql+ :fields [[:field {} 1]+ [:field {} 2]+ [:field {} "wow"]]+ :source-table 1}]})))))diff --git a/test/metabase/lib/walk_test.cljc b/test/metabase/lib/walk_test.cljcindex ea6732f5577..8f18451b425 100644--- a/test/metabase/lib/walk_test.cljc+++ b/test/metabase/lib/walk_test.cljc@@ -4,7 +4,9 @@:cljs ([metabase.test-runner.assert-exprs.approximately-equal]))[clojure.test :refer [are deftest is testing use-fixtures]][metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata][metabase.lib.test-metadata :as meta]+ [metabase.lib.util.match :as lib.util.match][metabase.lib.walk :as lib.walk][metabase.util :as u][metabase.util.malli.registry :as mr]))@@ -223,6 +225,28 @@60]]]@calls))))+(deftest ^:parallel walk-clauses-join-conditions-test+ (let [query (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))+ (lib/join (meta/table-metadata :categories)))]+ (is (=? {:stages [{:joins [{:alias "Categories"+ :conditions [[:=+ {}+ [:field {} (meta/id :venues :category-id)]+ [:field {:join-alias "Categories"} (meta/id :categories :id)]]]}]}]}+ query))+ (is (=? {:stages [{:joins [{:alias "Categories"+ :conditions [[:=+ {}+ [:field {} "CATEGORY_ID"]+ [:field {:join-alias "Categories"} "ID"]]]}]}]}+ (lib.walk/walk-clauses+ query+ (fn [query _path-type _stage-or-join-path clause]+ (lib.util.match/match-lite clause+ [:field opts id]+ (let [col (lib.metadata/field query id)]+ [:field (merge (select-keys col [:base-type]) opts) (:name col)]))))))))+(deftest ^:parallel walk-clauses-identity-test(testing "If we don't update any clauses then we should return the original query"(let [query (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))diff --git a/test/metabase/query_processor/middleware/add_implicit_joins_test.clj b/test/metabase/query_processor/middleware/add_implicit_joins_test.cljindex 9281ec464be..ed5126fd449 100644--- a/test/metabase/query_processor/middleware/add_implicit_joins_test.clj+++ b/test/metabase/query_processor/middleware/add_implicit_joins_test.clj@@ -10,7 +10,6 @@[metabase.query-processor :as qp][metabase.query-processor.middleware.add-implicit-clauses :as qp.add-implicit-clauses][metabase.query-processor.middleware.add-implicit-joins :as qp.add-implicit-joins]- [metabase.query-processor.middleware.add-source-metadata :as qp.add-source-metadata][metabase.query-processor.middleware.fetch-source-query][metabase.query-processor.schema :as qp.schema][metabase.test :as mt]@@ -352,7 +351,6 @@[:field %products.category {:source-field %product-id:source-field-name "Orders__PRODUCT_ID":source-field-join-alias "Card"}]]})- qp.add-source-metadata/add-source-metadata-for-source-queriesadd-implicit-clausesadd-implicit-joins)))))diff --git a/test/metabase/query_processor/middleware/add_remaps_test.clj b/test/metabase/query_processor/middleware/add_remaps_test.cljindex a3173bb00eb..59aaf37d99f 100644--- a/test/metabase/query_processor/middleware/add_remaps_test.clj+++ b/test/metabase/query_processor/middleware/add_remaps_test.clj@@ -693,10 +693,10 @@;; -- Cam;;;;- ;; 1 remap for self-joined orders.user-id => people.email- [:field (meta/id :people :email) {:join-alias "j"}];; 1 remap for source table orders.user-id => people.email- [:field (meta/id :people :email) {:join-alias "PEOPLE__via__USER_ID", ::qp.add-remaps/new-field-dimension-id pos-int?}]]}+ [:field (meta/id :people :email) {:join-alias "PEOPLE__via__USER_ID", ::qp.add-remaps/new-field-dimension-id pos-int?}]+ ;; 1 remap for self-joined orders.user-id => people.email+ [:field (meta/id :people :email) {:join-alias "j"}]]}preprocessed-query)))))))(deftest ^:parallel add-remaps-to-joins-e2e-test-2@@ -760,10 +760,8 @@[:field (meta/id :venues :longitude) {:join-alias "J"}][:field (meta/id :venues :price) {:join-alias "J"}];; we shouldn't use IDs here because they would be ambiguous.- [:field "NAME_2" {:join-alias "J"}]- [:field "NAME_3" {:join-alias "J"}]- ;; TODO -- this shouldn't be here! DUPLICATE!- [:field (meta/id :categories :name) {}]]}+ [:field (meta/id :categories :name) {:join-alias "J", :source-field (meta/id :venues :id)}]+ [:field (meta/id :categories :name) {:join-alias "J", :source-field (meta/id :venues :category-id)}]]};; these are in the opposite order as the join source query because `CATEGORY_ID`;; appears before `ID` here but the other way around above.{:alias "CATEGORIES__via__CATEGORY_ID"}@@ -777,15 +775,14 @@[:field (meta/id :venues :latitude) {:join-alias "J"}][:field (meta/id :venues :longitude) {:join-alias "J"}][:field (meta/id :venues :price) {:join-alias "J"}]- [:field "NAME_2" {:join-alias "J"}]- [:field "NAME_3" {:join-alias "J"}][:field (meta/id :categories :name) {:join-alias "CATEGORIES__via__CATEGORY_ID"}][:field (meta/id :categories :name) {:join-alias "CATEGORIES__via__ID"}]- ;; TODO DUPLICATE!!!- [:field (meta/id :categories :name) {:join-alias "J"}]]}}+ [:field (meta/id :categories :name) {:join-alias "J", :source-field (meta/id :venues :id)}]+ [:field (meta/id :categories :name) {:join-alias "J", :source-field (meta/id :venues :category-id)}]]}}(-> queryqp.preprocess/preprocess- lib/->legacy-MBQL))))))+ lib/->legacy-MBQL+ (select-keys [:query])))))))(deftest ^:parallel do-not-include-remaps-in-joins-for-columns-that-are-not-in-fields-e2e-test(testing "Do not include remaps in joins for columns that are not in :fields (#63165)"diff --git a/test/metabase/query_processor/middleware/add_source_metadata_test.clj b/test/metabase/query_processor/middleware/add_source_metadata_test.cljdeleted file mode 100644index 2b8d2724bd7..00000000000--- a/test/metabase/query_processor/middleware/add_source_metadata_test.clj+++ /dev/null@@ -1,382 +0,0 @@-(ns metabase.query-processor.middleware.add-source-metadata-test- (:require- [clojure.test :refer :all]- [medley.core :as m]- [metabase.driver :as driver]- [metabase.lib.test-metadata :as meta]- [metabase.lib.test-util :as lib.tu]- [metabase.lib.test-util.macros :as lib.tu.macros]- [metabase.query-processor.middleware.add-source-metadata :as add-source-metadata]- [metabase.query-processor.preprocess :as qp.preprocess]- [metabase.query-processor.store :as qp.store]- [metabase.test :as mt]- [metabase.util :as u]))--(defn- add-source-metadata [query]- (driver/with-driver :h2- (qp.store/with-metadata-provider (if (qp.store/initialized?)- (qp.store/metadata-provider)- meta/metadata-provider)- (add-source-metadata/add-source-metadata-for-source-queries query))))--(defn- results-col [col]- (select-keys- col- [:id :table_id :name :display_name :base_type :effective_type :coercion_strategy- :semantic_type :unit :fingerprint :settings :field_ref :nfc_path :parent_id]))--(defn- results-metadata [cols]- (map results-col cols))--(defn- venues-source-metadata- ([]- (venues-source-metadata :id :name :category-id :latitude :longitude :price))-- ([& field-names]- (let [field-ids (map #(meta/id :venues (keyword (u/lower-case-en (name %))))- field-names)]- (qp.store/with-metadata-provider (if (qp.store/initialized?)- (qp.store/metadata-provider)- meta/metadata-provider)- (results-metadata- (qp.preprocess/query->expected-cols- (lib.tu.macros/mbql-query venues- {:fields (for [id field-ids] [:field id nil])- :limit 1})))))))--(deftest ^:parallel basic-test- (testing "Can we automatically add source metadata to the parent level of a query? If the source query has `:fields`"- (is (=? (lib.tu.macros/mbql-query venues- {:source-query {:source-table $$venues- :fields [$id $name]}- :source-metadata (venues-source-metadata :id :name)})- (add-source-metadata- (lib.tu.macros/mbql-query venues- {:source-query {:source-table $$venues- :fields [$id $name]}}))))))--(deftest ^:parallel basic-parent-level-test- (testing (str "Can we automatically add source metadata to the parent level of a query? If the source query does not "- "have `:fields`")- (is (=? (lib.tu.macros/mbql-query venues- {:source-query {:source-table $$venues}- :source-metadata (venues-source-metadata)})- (add-source-metadata- (lib.tu.macros/mbql-query venues- {:source-query {:source-table $$venues}}))))))--(deftest ^:parallel basic-summary-columns-test- (testing "Can we add source metadata for a source query that has breakouts/aggregations?"- (is (=? (lib.tu.macros/mbql-query venues- {:source-query {:source-table $$venues- :aggregation [[:count]]- :breakout [$price]}- :source-metadata [(-> (venues-source-metadata :price)- first)- {:name "count"- :display_name "Count"- :base_type :type/Integer- :semantic_type :type/Quantity- :field_ref [:aggregation 0]}]})- (add-source-metadata- (lib.tu.macros/mbql-query venues- {:source-query {:source-table $$venues- :aggregation [[:count]]- :breakout [$price]}}))))))--(deftest ^:parallel basic-aggregation-with-field-test- (testing "Can we add source metadata for a source query that has an aggregation for a specific Field?"- (is (=? (lib.tu.macros/mbql-query venues- {:source-query {:source-table $$venues- :aggregation [[:avg $id]]- :breakout [$price]}- :source-metadata [(-> (venues-source-metadata :price)- first)- {:name "avg"- :display_name "Average of ID"- :base_type :type/Float- :semantic_type :type/PK- :field_ref [:aggregation 0]}]})- (add-source-metadata- (lib.tu.macros/mbql-query venues- {:source-query {:source-table $$venues- :aggregation [[:avg $id]]- :breakout [$price]}}))))))--(defn- source-metadata [query]- (get-in query [:query :source-metadata] query))--(deftest ^:parallel named-aggregations-test- (testing "adding source metadata for source queries with named aggregations"- (testing "w/ `:name` and `:display-name`"- (is (=? (lib.tu.macros/mbql-query venues- {:source-query {:source-table $$venues- :aggregation [[:aggregation-options- [:avg $id]- {:name "some_generated_name", :display-name "My Cool Ag"}]]- :breakout [$price]}- :source-metadata [(-> (venues-source-metadata :price)- first)- {:name "some_generated_name"- :display_name "My Cool Ag"- :base_type :type/Float- :semantic_type :type/PK- :field_ref [:aggregation 0]}]})- (add-source-metadata- (lib.tu.macros/mbql-query venues- {:source-query {:source-table $$venues- :aggregation [[:aggregation-options- [:avg $id]- {:name "some_generated_name", :display-name "My Cool Ag"}]]- :breakout [$price]}})))))))--(deftest ^:parallel named-aggregations-name-only-test- (testing "w/ `:name` only"- (is (=? [{:name "some_generated_name"- :display_name "Average of ID"- :base_type :type/Float- :semantic_type :type/PK- :field_ref [:aggregation 0]}]- (source-metadata- (add-source-metadata- (lib.tu.macros/mbql-query venues- {:source-query {:source-table $$venues- :aggregation [[:aggregation-options [:avg $id] {:name "some_generated_name"}]]}})))))))--(deftest ^:parallel named-aggregations-display-name-only-test- (testing "w/ `:display-name` only"- (is (=? [{:name "avg"- :display_name "My Cool Ag"- :base_type :type/Float- :semantic_type :type/PK- :field_ref [:aggregation 0]}]- (source-metadata- (add-source-metadata- (lib.tu.macros/mbql-query venues- {:source-query {:source-table $$venues- :aggregation [[:aggregation-options [:avg $id] {:display-name "My Cool Ag"}]]}})))))))--(deftest ^:parallel nested-sources-test- (testing (str "Can we automatically add source metadata to the parent level of a query? If the source query has a "- "source query with source metadata")- (is (=? (lib.tu.macros/mbql-query venues- {:source-query {:source-query {:source-table $$venues- :fields [$id $name]}- :source-metadata (venues-source-metadata :id :name)}- :source-metadata (venues-source-metadata :id :name)})- (add-source-metadata- (lib.tu.macros/mbql-query venues- {:source-query {:source-query {:source-table $$venues- :fields [$id $name]}- :source-metadata (venues-source-metadata :id :name)}}))))))--(deftest ^:parallel nested-sources-3-levels-with-source-metadata-test- (testing "Can we automatically add source metadata if a source-query nested 3 levels has `:source-metadata`?"- (is (=? (lib.tu.macros/mbql-query venues- {:source-query {:source-query {:source-query {:source-table $$venues- :fields [$id $name]}- :source-metadata (venues-source-metadata :id :name)}- :source-metadata (venues-source-metadata :id :name)}- :source-metadata (venues-source-metadata :id :name)})- (add-source-metadata- (lib.tu.macros/mbql-query venues- {:source-query- {:source-query- {:source-query {:source-table $$venues- :fields [$id $name]}- :source-metadata (venues-source-metadata :id :name)}}}))))))--(deftest ^:parallel nested-sources-3-levels-with-no-source-metadata-test- (testing "Ok, how about a source query nested 3 levels with no `source-metadata`?"- (is (=? (lib.tu.macros/mbql-query venues- {:source-query {:source-query {:source-query {:source-table $$venues}- :source-metadata (venues-source-metadata)}- :source-metadata (venues-source-metadata)}- :source-metadata (venues-source-metadata)})- (add-source-metadata- (lib.tu.macros/mbql-query venues- {:source-query {:source-query {:source-query {:source-table $$venues}}}}))))))--(deftest ^:parallel nested-sources-3-levels-with-fields-test- (testing "nested 3 levels with `fields`"- (is (=? (lib.tu.macros/mbql-query venues- {:source-query {:source-query {:source-query {:source-table $$venues- :fields [$id $name]}- :source-metadata (venues-source-metadata :id :name)}- :source-metadata (venues-source-metadata :id :name)}- :source-metadata (venues-source-metadata :id :name)})- (add-source-metadata- (lib.tu.macros/mbql-query venues- {:source-query {:source-query {:source-query {:source-table $$venues- :fields [$id $name]}}}}))))))--(deftest ^:parallel nested-sources-3-levels-with-summary-columns-test- (testing "nested 3 levels with breakouts/aggregations"- (qp.store/with-metadata-provider meta/metadata-provider- ;; field ref for the count aggregation differs slightly depending on what level of the query we're at; at the- ;; most-deeply-nested level we can use the `[:aggregation 0]` ref to refer to it; at higher levels we have to- ;; refer to it with a field literal- (is (=? (letfn [(metadata-with-count-field-ref [field-ref]- [(-> (venues-source-metadata :price) first)- (let [[count-col] (->> (lib.tu.macros/mbql-query venues- {:aggregation [[:count]]})- qp.preprocess/query->expected-cols- results-metadata)]- (-> count-col- (dissoc :effective_type)- (assoc :field_ref field-ref- :base_type :type/Integer)))])]- (lib.tu.macros/mbql-query venues- {:source-query {:source-query {:source-query {:source-table $$venues- :aggregation [[:count]]- :breakout [$price]}- :source-metadata (metadata-with-count-field-ref [:aggregation 0])}- :source-metadata (metadata-with-count-field-ref *count/Integer)}- :source-metadata (metadata-with-count-field-ref *count/Integer)}))- (add-source-metadata- (lib.tu.macros/mbql-query venues- {:source-query {:source-query {:source-query {:source-table $$venues- :aggregation [[:count]]- :breakout [$price]}}}})))))))--(deftest ^:parallel nested-sources-with-source-native-query-test- (testing "can we add `source-metadata` to the parent level if the source query has a native source query, but itself has `source-metadata`?"- (is (=? (lib.tu.macros/mbql-query venues- {:source-query {:source-query {:native "SELECT \"ID\", \"NAME\" FROM \"VENUES\";"}- :source-metadata (venues-source-metadata :id :name)}- :source-metadata (venues-source-metadata :id :name)})- (add-source-metadata- (lib.tu.macros/mbql-query venues- {:source-query {:source-query {:native "SELECT \"ID\", \"NAME\" FROM \"VENUES\";"}- :source-metadata (venues-source-metadata :id :name)}}))))))--(deftest ^:parallel joins-test- (testing "should work inside JOINS as well"- (is (=? (lib.tu.macros/mbql-query venues- {:source-table $$venues- :joins [{:source-query {:source-table $$venues- :fields [$id $name]}- :source-metadata (venues-source-metadata :id :name)}]})- (add-source-metadata- (lib.tu.macros/mbql-query venues- {:source-table $$venues- :joins [{:source-query {:source-table $$venues- :fields [$id $name]}- :condition [:= "A" "B"]}]}))))))--(deftest ^:parallel binned-fields-test- (testing "source metadata should handle source queries that have binned fields"- (qp.store/with-metadata-provider (lib.tu/mock-metadata-provider- meta/metadata-provider- {:settings {:breakout-bin-width 5.0}})- (is (=? (lib.tu.macros/mbql-query venues- {:source-query {:source-table $$venues- :aggregation [[:count]]- :breakout [[:field %latitude {:binning {:strategy :default}}]]}- :source-metadata (concat- (let [[lat-col] (venues-source-metadata :latitude)- [count-col] (results-metadata (qp.preprocess/query->expected-cols- (lib.tu.macros/mbql-query venues- {:aggregation [[:count]]})))]- [(assoc lat-col- :field_ref [:field- (meta/id :venues :latitude)- {:binning {:strategy :bin-width- :min-value 10.0- :max-value 45.0- :num-bins 7- :bin-width 5.0}}]- :display_name "Latitude: 5°")- ;; computed column doesn't have an effective type in middleware before query- (-> count-col- (dissoc :effective_type)- ;; the type that comes back from H2 is :type/BigInteger but the type that comes- ;; back from calculating it with MLv2 is just plain :type/Integer- (assoc :base_type :type/Integer))]))})- (add-source-metadata- (lib.tu.macros/mbql-query venues- {:source-query- {:source-table $$venues- :aggregation [[:count]]- :breakout [[:field %latitude {:binning {:strategy :default}}]]}})))))))--(deftest ^:parallel deduplicate-column-names-test- (testing "Metadata that gets added to source queries should have deduplicated column names"- (let [query (add-source-metadata- (lib.tu.macros/mbql-query checkins- {:source-query {:source-table $$checkins- :joins [{:fields :all- :alias "u"- :source-table $$users- :condition [:= $user-id &u.users.id]}]}- :joins [{:fields :all- :alias "v"- :source-table $$venues- :condition [:= *user-id &v.venues.id]}]- :order-by [[:asc $id]]- :limit 2}))]- (is (= ["ID" "DATE" "USER_ID" "VENUE_ID" "ID_2" "NAME" "LAST_LOGIN"]- (map :name (get-in query [:query :source-metadata])))))))--(deftest ^:parallel inception-test- (testing "Should be able to do an 'inception-style' nesting of source > source > source with a join (#14724)"- ;; these tests look at the metadata for just one column so it's easier to spot the differences.- (letfn [(ean-metadata [query]- (as-> query query- (get-in query [:query :source-metadata])- (m/index-by :name query)- (get query "EAN")- (select-keys query [:name :display_name :base_type :semantic_type :id :field_ref])))]- (let [base-query (lib.tu.macros/mbql-query orders- {:source-table $$orders- :joins [{:fields :all- :source-table $$products- :condition [:= $product-id &Products.products.id]- :alias "Products"}]- :limit 10})]- (testing "Make sure metadata is correct for the 'EAN' column with"- (doseq [level (range 1 4)- ;; existing usage, do not use this going forward- :let [query #_{:clj-kondo/ignore [:deprecated-var]} (mt/nest-query base-query level)]]- (testing (format "%d level(s) of nesting" level)- (is (= (lib.tu.macros/$ids products- {:name "EAN"- :display_name "Products → Ean"- :base_type :type/Text- :id %ean- :field_ref &Products.ean})- (ean-metadata (add-source-metadata query)))))))))))--(deftest ^:parallel ignore-legacy-source-metadata-test- (testing "Should ignore 'legacy' < 0.38.0 source metadata and recalculate it for MBQL queries (#14788)"- ;; normally this middleware will use existing source metadata rather than recalculating it, but if we encounter <- ;; 0.38.0 source metadata that is missing `:field_ref` and `:id` information we should ignore it.- (let [query (lib.tu.macros/mbql-query orders- {:source-query {:source-table $$orders- :joins [{:source-table $$products- :alias "ℙ"- :fields :all- :condition [:= $product-id &ℙ.products.id]}]- :order-by [[:asc $id]]- :limit 2}})- metadata (qp.store/with-metadata-provider meta/metadata-provider- (qp.preprocess/query->expected-cols query))- ;; the actual metadata this middleware should return. Doesn't have all the columns that come back from- ;; `qp.preprocess/query->expected-cols`- expected-metadata (map results-col metadata)]- (letfn [(added-metadata [query]- (get-in (add-source-metadata query) [:query :source-metadata]))]- (testing "\nShould add source metadata if there's none already"- (is (=? expected-metadata- (added-metadata query))))- (testing "\nShould use existing metadata if it's already there"- ;; since it's using the existing metadata, it should have all the extra keys instead of the subset in- ;; `expected-metadata`- (is (= metadata- (added-metadata (assoc-in query [:query :source-metadata] metadata)))))- (testing "\nShould ignore legacy metadata"- ;; pre-0.38.0 metadata didn't have `field_ref` or `id.`- (let [legacy-metadata (for [col metadata]- (dissoc col :field_ref :id))]- (is (=? expected-metadata- (added-metadata (assoc-in query [:query :source-metadata] legacy-metadata))))))))))diff --git a/test/metabase/query_processor/middleware/resolve_fields_test.clj b/test/metabase/query_processor/middleware/resolve_fields_test.cljdeleted file mode 100644index eac6dffc742..00000000000--- a/test/metabase/query_processor/middleware/resolve_fields_test.clj+++ /dev/null@@ -1,18 +0,0 @@-(ns metabase.query-processor.middleware.resolve-fields-test- (:require- [clojure.test :refer :all]- [metabase.query-processor.middleware.resolve-fields :as qp.resolve-fields]- [metabase.util.malli.fn :as mu.fn]))--(deftest ^:parallel field-ids-test- (testing "Field IDs in metadata should get picked up correctly"- (binding [mu.fn/*enforce* false]- (is (= #{1}- (#'qp.resolve-fields/field-ids- {:lib/type :mbql/query- :database 1- :stages [{:lib/type :mbql.stage/mbql- :lib/stage-metadata {:lib/type :metadata/results- :columns [{:lib/type :metadata/column, :id 1}- {:lib/type :metadata/column, :id 0}]}- :source-table 1}]}))))))diff --git a/test/metabase/query_processor/middleware/resolve_joins_test.clj b/test/metabase/query_processor/middleware/resolve_joins_test.cljindex 5ed2ad16aa1..4b939e49ba6 100644--- a/test/metabase/query_processor/middleware/resolve_joins_test.clj+++ b/test/metabase/query_processor/middleware/resolve_joins_test.clj@@ -6,144 +6,132 @@[metabase.lib.test-metadata :as meta][metabase.lib.test-util :as lib.tu][metabase.lib.test-util.macros :as lib.tu.macros]- [metabase.query-processor :as qp][metabase.query-processor.middleware.resolve-joins :as resolve-joins]- [metabase.query-processor.preprocess :as qp.preprocess]- [metabase.query-processor.store :as qp.store]- [metabase.test :as mt]))+ [metabase.query-processor.preprocess :as qp.preprocess]))-(defn- resolve-joins [query]- (if (qp.store/initialized?)- (resolve-joins/resolve-joins query)- (qp.store/with-metadata-provider (mt/id)- (resolve-joins/resolve-joins query))))+(defn- resolve-joins+ ([query]+ (resolve-joins meta/metadata-provider query))++ ([mp query]+ (if (:lib/type query)+ (resolve-joins/resolve-joins query)+ (-> (lib/query mp query)+ resolve-joins/resolve-joins+ lib/->legacy-MBQL))))(deftest ^:parallel joins->fields-test- (is (= [[:field 1 {:join-alias "B", :qp/ignore-coercion true}]- [:field 2 {:join-alias "B", :qp/ignore-coercion true}]- [:field 3 {:join-alias "C", :qp/ignore-coercion true}]- [:field 4 {:join-alias "C", :qp/ignore-coercion true}]]- (#'resolve-joins/joins->fields [{:alias "A"- :fields :all}- {:alias "B"- :fields [[:field 1 {:join-alias "B"}]- ;; missing join alias- [:field 2 nil]]}- {:alias "C"- :fields [[:field 3 {:join-alias "C"}]- ;; wrong join alias- [:field 4 {:join-alias "X"}]]}]))))+ (is (=? [[:field {:join-alias "B", :qp/ignore-coercion true} 1]+ [:field {:join-alias "B", :qp/ignore-coercion true} 2]+ [:field {:join-alias "C", :qp/ignore-coercion true} 3]+ [:field {:join-alias "C", :qp/ignore-coercion true} 4]]+ (#'resolve-joins/joins->fields [{:alias "A"+ :fields :all}+ {:alias "B"+ :fields [[:field {:lib/uuid "00000000-0000-0000-0000-000000000000", :join-alias "B"} 1]+ ;; missing join alias+ [:field {:lib/uuid "00000000-0000-0000-0000-000000000001"} 2]]}+ {:alias "C"+ :fields [[:field {:lib/uuid "00000000-0000-0000-0000-000000000002", :join-alias "C"} 3]+ ;; wrong join alias+ [:field {:lib/uuid "00000000-0000-0000-0000-000000000003", :join-alias "X"} 4]]}]))))(deftest ^:parallel no-op-test(testing "Does the middleware function if the query has no joins?"- (is (= (mt/mbql-query venues)+ (is (= (lib.tu.macros/mbql-query venues)(resolve-joins- (mt/mbql-query venues))))))+ (lib.tu.macros/mbql-query venues))))))(deftest ^:parallel fields-none-test(testing "Can we resolve some joins w/ fields = none?"- (is (=? (mt/mbql-query venues+ (is (=? (lib.tu.macros/mbql-query venues{:joins[{:source-table $$categories:alias "c":strategy :left-join- :condition [:= $category_id &c.categories.id]}]})+ :condition [:= $category-id &c.categories.id]}]})(resolve-joins- (mt/mbql-query venues+ (lib.tu.macros/mbql-query venues{:joins [{:source-table $$categories:alias "c"- :condition [:= $category_id &c.categories.id]+ :condition [:= $category-id &c.categories.id]:fields :none}]}))))))(deftest ^:parallel fields-all-test(testing "Can we resolve some joins w/ fields = all ???"- (is (= (mt/mbql-query venues- {:joins- [{:source-table $$categories- :alias "c"- :strategy :left-join- :condition [:= $category_id &c.categories.id]- :fields [&c.categories.id- &c.categories.name]}]- :fields [$venues.id- $venues.name- [:field %categories.id {:join-alias "c"- :qp/ignore-coercion true}]- [:field %categories.name {:join-alias "c"- :qp/ignore-coercion true}]]})- (resolve-joins- (mt/mbql-query venues- {:fields [$venues.id $venues.name]- :joins [{:source-table $$categories- :alias "c"- :condition [:= $category_id &c.categories.id]- :fields :all}]}))))))+ (is (=? (lib.tu.macros/mbql-query venues+ {:joins+ [{:source-table $$categories+ :alias "c"+ :strategy :left-join+ :condition [:= $category-id &c.categories.id]+ :fields [&c.categories.id+ &c.categories.name]}]+ :fields [$venues.id+ $venues.name+ [:field %categories.id {:join-alias "c"+ :qp/ignore-coercion true}]+ [:field %categories.name {:join-alias "c"+ :qp/ignore-coercion true}]]})+ (resolve-joins+ (lib.tu.macros/mbql-query venues+ {:fields [$venues.id $venues.name]+ :joins [{:source-table $$categories+ :alias "c"+ :condition [:= $category-id &c.categories.id]+ :fields :all}]}))))))(deftest ^:parallel fields-sequence-test(testing "can we resolve joins w/ fields = <sequence>"- (is (= (mt/mbql-query venues+ (is (= (lib.tu.macros/mbql-query venues{:joins[{:source-table $$categories:alias "c":strategy :left-join- :condition [:= $category_id &c.categories.id]+ :condition [:= $category-id &c.categories.id]:fields [&c.categories.name]}]:fields [$venues.id$venues.name[:field %categories.name {:join-alias "c":qp/ignore-coercion true}]]})(resolve-joins- (mt/mbql-query venues+ (lib.tu.macros/mbql-query venues{:fields [$venues.id $venues.name]:joins [{:source-table $$categories:alias "c"- :condition [:= $category_id &c.categories.id]+ :condition [:= $category-id &c.categories.id]:fields [&c.categories.name]}]}))))))(deftest ^:parallel join-table-without-alias-test(testing "Does joining a table an explicit alias add a default alias?"- (is (=? (mt/mbql-query venues+ (is (=? (lib.tu.macros/mbql-query venues{:joins [{:source-table $$categories:alias "__join":strategy :left-join- :condition [:= $category_id 1]}+ :condition [:= $category-id 1]}{:source-table $$categories:alias "__join_2":strategy :left-join- :condition [:= $category_id 2]}]- :source-table (mt/id :venues)})+ :condition [:= $category-id 2]}]+ :source-table (meta/id :venues)})(resolve-joins- (mt/mbql-query venues+ (lib.tu.macros/mbql-query venues{:joins [{:source-table $$categories- :condition [:= $category_id 1]}+ :condition [:= $category-id 1]}{:source-table $$categories- :condition [:= $category_id 2]}]}))))))--(deftest ^:parallel disallow-joins-against-table-on-different-db-test- (testing "Test that joining against a table in a different DB throws an Exception"- (qp.store/with-metadata-provider (lib.tu/mock-metadata-provider- {:database meta/database- :tables [(meta/table-metadata :venues)]})- (is (thrown-with-msg?- clojure.lang.ExceptionInfo- #"\QFailed to fetch :metadata/table\E"- (resolve-joins- (mt/mbql-query venues- {:joins [{:source-table (meta/id :categories)- :alias "t"- :condition [:= $category_id 1]}]})))))))+ :condition [:= $category-id 2]}]}))))))(deftest ^:parallel resolve-explicit-joins-when-implicit-joins-are-present-test(testing "test that resolving explicit joins still works if implict joins are present"- (is (=? (mt/mbql-query checkins+ (is (=? (lib.tu.macros/mbql-query checkins{:source-table $$checkins:aggregation [[:sum &USERS__via__USER_ID.users.id]]:breakout [$id]:joins [{:source-table $$users:alias "USERS__via__USER_ID":strategy :left-join- :condition [:= $user_id &USERS__via__USER_ID.users.id]- :fk-field-id (mt/id :checkins :user_id)}+ :condition [:= $user-id &USERS__via__USER_ID.users.id]+ :fk-field-id (meta/id :checkins :user-id)}{:alias "u":source-table $$users:strategy :left-join@@ -152,15 +140,15 @@&u.users.id]}]:limit 10})(resolve-joins- (mt/mbql-query checkins+ (lib.tu.macros/mbql-query checkins{:source-table $$checkins:aggregation [[:sum &USERS__via__USER_ID.users.id]]:breakout [$id]:joins [{:source-table $$users:alias "USERS__via__USER_ID":strategy :left-join- :condition [:= $user_id &USERS__via__USER_ID.users.id]- :fk-field-id (mt/id :checkins :user_id)+ :condition [:= $user-id &USERS__via__USER_ID.users.id]+ :fk-field-id (meta/id :checkins :user-id):fields :none}{:alias "u":source-table $$users@@ -171,98 +159,96 @@(deftest ^:parallel join-with-source-query-test(testing "Does a join using a source query get its Tables resolved?"- (is (=? (mt/mbql-query venues+ (is (=? (lib.tu.macros/mbql-query venues{:joins [{:alias "cat"- :source-query {:source-table $$categories}+ :source-table $$categories:strategy :left-join:condition [:=- $category_id+ $category-id[:field "ID" {:base-type :type/BigInteger, :join-alias "cat"}]]}]:order-by [[:asc $name]]:limit 3})(resolve-joins- (mt/mbql-query venues+ (lib.tu.macros/mbql-query venues{:joins [{:alias "cat":source-query {:source-table $$categories}:condition [:=- $category_id+ $category-id[:field "ID" {:base-type :type/BigInteger, :join-alias "cat"}]]}]:order-by [[:asc $name]]:limit 3}))))))(deftest ^:parallel resolve-source-query-with-fields-all-test(testing "Can we resolve joins using a `:source-query` and `:fields` `:all`?"- (let [source-metadata (get-in (qp/process-query- (qp/userland-query- (mt/mbql-query categories {:limit 1})))- [:data :results_metadata :columns])+ (let [source-metadata (qp.preprocess/query->expected-cols+ (lib/query meta/metadata-provider (lib.tu.macros/mbql-query categories {:limit 1})))resolved (resolve-joins- (mt/mbql-query venues+ (lib.tu.macros/mbql-query venues{:joins [{:alias "cat":source-query {:source-table $$categories}:source-metadata source-metadata:fields :all:condition [:=- $category_id+ $category-id[:field "ID" {:base-type :type/BigInteger, :join-alias "cat"}]]}]:order-by [[:asc $name]]:limit 3}))]- (is (= (mt/mbql-query venues- {:fields [[:field (mt/id :categories :id) {:join-alias "cat", :qp/ignore-coercion true}]- [:field (mt/id :categories :name) {:join-alias "cat", :qp/ignore-coercion true}]]- :joins [{:alias "cat"- :source-query {:source-table $$categories}- :source-metadata source-metadata- :strategy :left-join- :condition [:= $category_id [:field "ID" {:base-type :type/BigInteger, :join-alias "cat"}]]- :fields [&cat.categories.id- &cat.categories.name]}]- :order-by [[:asc $name]]- :limit 3})- resolved)))))+ (is (=? (lib.tu.macros/mbql-query venues+ {:fields [[:field (meta/id :categories :id) {:join-alias "cat", :qp/ignore-coercion true}]+ [:field (meta/id :categories :name) {:join-alias "cat", :qp/ignore-coercion true}]]+ :joins [{:alias "cat"+ :source-table $$categories+ :source-metadata source-metadata+ :strategy :left-join+ :condition [:= $category-id [:field "ID" {:base-type :type/BigInteger, :join-alias "cat"}]]+ :fields [&cat.categories.id+ &cat.categories.name]}]+ :order-by [[:asc $name]]+ :limit 3})+ resolved)))))(deftest ^:parallel dont-append-fields-if-parent-has-breakout-or-aggregation-test(testing "if the parent level has a breakout or aggregation, we shouldn't append Join fields to the parent level"- (is (= (mt/mbql-query users- {:joins [{:source-table $$checkins- :alias "c"- :strategy :left-join- :condition [:= $id [:field "USER_ID" {:base-type :type/Integer, :join-alias "c"}]]- :fields [&c.checkins.id- &c.checkins.date- &c.checkins.user_id- &c.checkins.venue_id]}]- :aggregation [[:sum [:field "id" {:base-type :type/Float, :join-alias "c"}]]]- :breakout [[:field %last_login {:temporal-unit :month}]]})- (resolve-joins- (mt/mbql-query users- {:joins [{:fields :all+ (is (=? (lib.tu.macros/mbql-query users+ {:joins [{:source-table $$checkins:alias "c"- :source-table $$checkins- :condition [:= $id [:field "USER_ID" {:base-type :type/Integer, :join-alias "c"}]]}]+ :strategy :left-join+ :condition [:= $id [:field "USER_ID" {:base-type :type/Integer, :join-alias "c"}]]+ :fields [&c.checkins.id+ &c.checkins.date+ &c.checkins.user-id+ &c.checkins.venue-id]}]:aggregation [[:sum [:field "id" {:base-type :type/Float, :join-alias "c"}]]]- :breakout [[:field %last_login {:temporal-unit :month}]]}))))))+ :breakout [[:field %last-login {:temporal-unit :month}]]})+ (resolve-joins+ (lib.tu.macros/mbql-query users+ {:joins [{:fields :all+ :alias "c"+ :source-table $$checkins+ :condition [:= $id [:field "USER_ID" {:base-type :type/Integer, :join-alias "c"}]]}]+ :aggregation [[:sum [:field "id" {:base-type :type/Float, :join-alias "c"}]]]+ :breakout [[:field %last-login {:temporal-unit :month}]]}))))))(deftest ^:parallel aggregation-field-ref-test(testing "Should correctly handle [:aggregation n] field refs"(is (some? (resolve-joins- (mt/mbql-query users+ (lib.tu.macros/mbql-query users{:fields [$id$name- [:field %last_login {:temporal-unit :default}]]+ [:field %last-login {:temporal-unit :default}]]:joins [{:fields :all:alias "__alias__"- :condition [:= $id [:field %checkins.user_id {:join-alias "__alias__"}]]+ :condition [:= $id [:field %checkins.user-id {:join-alias "__alias__"}]]:source-query {:source-table $$checkins:aggregation [[:sum $checkins.id]]- :breakout [$checkins.user_id]}+ :breakout [$checkins.user-id]}:source-metadata[{:name "USER_ID":display_name "User ID":base_type :type/Integer:semantic_type :type/FK- :id %checkins.user_id- :field_ref $checkins.user_id+ :id %checkins.user-id+ :field_ref $checkins.user-id:fingerprint {:global {:distinct-count 15, :nil% 0.0}}}{:name "sum":display_name "Sum of ID"@@ -274,14 +260,13 @@(deftest ^:parallel native-model-field-ref-test(testing "should use name-based field refs for joined native models with mapped database fields (metabase#58829)"- (let [source-metadata [{:id (mt/id :checkins :id)- :name "_USER_ID"+ (let [source-metadata [{:name "_USER_ID":display_name "User ID":base_type :type/Integer:semantic_type :type/FK:field_ref [:field "_USER_ID" {:base-type :type/Integer :join-alias "alias"}]:fingerprint {:global {:distinct-count 15, :nil% 0.0}}}]]- (is (= (mt/mbql-query users+ (is (= (lib.tu.macros/mbql-query users{:fields [$id[:field "_USER_ID" {:base-type :type/Integer:join-alias "alias"@@ -294,7 +279,7 @@:source-metadata source-metadata}]:limit 10})(resolve-joins- (mt/mbql-query users+ (lib.tu.macros/mbql-query users{:fields [$id]:joins [{:fields :all:alias "alias"@@ -304,31 +289,31 @@:limit 10})))))))(deftest ^:parallel join-against-source-query-test- (is (= (mt/mbql-query venues+ (is (= (lib.tu.macros/mbql-query venues{:joins [{:source-query {:source-table $$categories:fields [$categories.id$categories.name]}:alias "cat"- :condition [:= $venues.category_id &cat.*ID/BigInteger]+ :condition [:= $venues.category-id &cat.*ID/BigInteger]:strategy :left-join}]:fields [$venues.id$venues.name- $venues.category_id+ $venues.category-id$venues.latitude$venues.longitude$venues.price]:order-by [[:asc $venues.name]]:limit 3})(resolve-joins- (mt/mbql-query venues+ (lib.tu.macros/mbql-query venues{:joins [{:source-query {:source-table $$categories:fields [$categories.id$categories.name]}:alias "cat"- :condition [:= $venues.category_id &cat.*ID/BigInteger]}]+ :condition [:= $venues.category-id &cat.*ID/BigInteger]}]:fields [$venues.id$venues.name- $venues.category_id+ $venues.category-id$venues.latitude$venues.longitude$venues.price]@@ -337,27 +322,28 @@(deftest ^:parallel do-not-duplicate-columns-with-default-temporal-bucketing-test(testing "Do not add a duplicate column from a join if it uses :default temporal bucketing"- (let [original-query {:fields [[:field (meta/id :orders :id) nil]- [:field (meta/id :people :birth-date) {:join-alias "P", :temporal-unit :default}]]}]+ (let [original-stage {:fields [[:field {:lib/uuid "00000000-0000-0000-0000-000000000000"}+ (meta/id :orders :id)]+ [:field {:lib/uuid "00000000-0000-0000-0000-000000000001", :join-alias "P", :temporal-unit :default}+ (meta/id :people :birth-date)]]}](doseq [temporal-unit [nil :default]base-type [nil :type/Date]effective-type [nil :type/Date]inherited-temporal-unit [nil :default];; make sure random keys don't affect this either- nonsense-key [nil 1337]lib-key [nil "PRODUCTS"]- :let [opts (cond-> {:join-alias "P"}+ :let [opts (cond-> {:join-alias "P"+ :lib/uuid "00000000-0000-0000-0000-000000000002"}temporal-unit (assoc :temporal-unit temporal-unit)base-type (assoc :base-type base-type)effective-type (assoc :effective-type effective-type)inherited-temporal-unit (assoc :inherited-temporal-unit inherited-temporal-unit)- nonsense-key (assoc :nonsense-key nonsense-key)lib-key (assoc :lib/nonsense-key lib-key))- clause [:field (meta/id :people :birth-date) opts]]]+ clause [:field opts (meta/id :people :birth-date)]]](testing (pr-str clause)- (is (= original-query+ (is (= original-stage(resolve-joins/append-join-fields-to-fields- original-query+ original-stage[clause]))))))))(deftest ^:parallel do-not-duplicate-columns-with-default-temporal-bucketing-e2e-testdiff --git a/test/metabase/query_processor/middleware/resolve_source_table_test.clj b/test/metabase/query_processor/middleware/resolve_source_table_test.cljindex 3d1d1cbbd4c..c3d716c37db 100644--- a/test/metabase/query_processor/middleware/resolve_source_table_test.clj+++ b/test/metabase/query_processor/middleware/resolve_source_table_test.clj@@ -6,8 +6,8 @@[metabase.lib.metadata.protocols :as lib.metadata.protocols][metabase.lib.test-metadata :as meta][metabase.lib.test-util :as lib.tu]- [metabase.query-processor.middleware.resolve-source-table :as qp.resolve-source-table]- [metabase.test :as mt]))+ [metabase.lib.test-util.macros :as lib.tu.macros]+ [metabase.query-processor.middleware.resolve-source-table :as qp.resolve-source-table]))(defn- cached-metadata"Fetch the names of all the objects currently in the QP Store."@@ -15,12 +15,12 @@(let [tables [:venues :categories :users :checkins]]{:tables (into #{}(keep (fn [table]- (:name (lib.metadata.protocols/cached-metadata metadata-provider :metadata/table (mt/id table)))))+ (:name (lib.metadata.protocols/cached-metadata metadata-provider :metadata/table (meta/id table)))))tables)}))(defn- resolve-and-return-cached-metadata([query]- (resolve-and-return-cached-metadata (mt/metadata-provider) query))+ (resolve-and-return-cached-metadata (lib.metadata.cached-provider/cached-metadata-provider meta/metadata-provider) query))([metadata-provider query](let [query (lib/query metadata-provider query)]@@ -30,7 +30,7 @@(deftest ^:parallel basic-test(testing "does `resolve-source-tables` resolve source tables?"(is (= {:tables #{"VENUES"}}- (resolve-and-return-cached-metadata (mt/mbql-query venues))))))+ (resolve-and-return-cached-metadata (lib.tu.macros/mbql-query venues))))))(deftest ^:parallel validate-database-test(testing "If the Table does not belong to the current Database, does it throw an Exception?"@@ -56,7 +56,7 @@clojure.lang.ExceptionInfo#"Invalid output:.*should be a positive int, got: 0"(resolve-and-return-cached-metadata- {:database (mt/id)+ {:database (meta/id):type :query:query {:source-table 0}})))))@@ -64,49 +64,65 @@(testing "Does `resolve-source-tables` resolve source tables in nested source queries?"(is (= {:tables #{"VENUES"}}(resolve-and-return-cached-metadata- (mt/mbql-query nil+ (lib.tu.macros/mbql-query nil{:source-query {:source-table $$venues}}))))(is (= {:tables #{"VENUES"}}(resolve-and-return-cached-metadata- (mt/mbql-query nil+ (lib.tu.macros/mbql-query nil{:source-query {:source-query {:source-table $$venues}}}))))))(deftest ^:parallel joins-test(testing "Does `resolve-source-tables` resolve source tables in joins?"(is (= {:tables #{"CATEGORIES" "VENUES"}}(resolve-and-return-cached-metadata- (mt/mbql-query venues+ (lib.tu.macros/mbql-query venues{:joins [{:source-table $$categories:alias "c"- :condition [:= $category_id &c.categories.id]}]}))))))+ :condition [:= $category-id &c.categories.id]}]}))))))(deftest ^:parallel joins-in-nested-queries-test(testing "Does `resolve-source-tables` resolve source tables in joins inside nested source queries?"(is (= {:tables #{"CATEGORIES" "VENUES"}}(resolve-and-return-cached-metadata- (mt/mbql-query venues+ (lib.tu.macros/mbql-query venues{:source-query {:source-table $$venues:joins [{:source-table $$categories:alias "c"- :condition [:= $category_id &c.categories.id]}]}}))))))+ :condition [:= $category-id &c.categories.id]}]}}))))))(deftest ^:parallel nested-queries-in-joins-test(testing "Does `resolve-source-tables` resolve source tables inside nested source queries inside joins?"(is (= {:tables #{"CATEGORIES" "VENUES"}}(resolve-and-return-cached-metadata- (mt/mbql-query venues+ (lib.tu.macros/mbql-query venues{:joins [{:source-query {:source-table $$categories}:alias "c"- :condition [:= $category_id &c.categories.id]}]}))))))+ :condition [:= $category-id &c.categories.id]}]}))))))(deftest ^:parallel nested-queries-in-joins-in-nested-queries-test(testing (str "Does `resolve-source-tables` resolve source tables inside nested source queries inside joins inside ""nested source queries?")(is (= {:tables #{"CATEGORIES" "VENUES"}}(resolve-and-return-cached-metadata- (mt/mbql-query venues+ (lib.tu.macros/mbql-query venues{:source-query {:source-table $$venues:joins [{:source-query {:source-table $$categories}:alias "c"- :condition [:= $category_id &c.categories.id]}]}}))))))+ :condition [:= $category-id &c.categories.id]}]}}))))))++(deftest ^:parallel disallow-joins-against-table-on-different-db-test+ (testing "Test that joining against a table in a different DB throws an Exception"+ (let [mp (lib.tu/mock-metadata-provider+ {:database meta/database+ :tables [(meta/table-metadata :venues)]})]+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"\QFailed to fetch :metadata/table\E"+ (qp.resolve-source-table/resolve-source-tables+ (lib/query+ mp+ (lib.tu.macros/mbql-query venues+ {:joins [{:source-table (meta/id :categories)+ :alias "t"+ :condition [:= $category-id 1]}]}))))))))diff --git a/test/metabase/query_processor/preprocess_test.clj b/test/metabase/query_processor/preprocess_test.cljindex b3ea18e1ca8..797cc05e96d 100644--- a/test/metabase/query_processor/preprocess_test.clj+++ b/test/metabase/query_processor/preprocess_test.clj@@ -2,7 +2,6 @@(:require[clojure.string :as str][clojure.test :refer :all]- [medley.core :as m][metabase.driver :as driver][metabase.lib.card :as lib.card][metabase.lib.card-test]@@ -233,21 +232,6 @@{:source-table "card__2":aggregation [[:sum [:field "TOTAL" {:base-type :type/Float}]]]:breakout [[:field "RATING" {:base-type :type/Float}]]}))]- (let [preprocessed (qp.preprocess/preprocess query)- stages (:stages preprocessed)]- (testing "added metadata"- (testing "first stage (from Card 1)"- (is (=? {:name "RATING"- :display-name "Product → Rating"- :metabase.lib.join/join-alias "Product"}- (m/find-first #(= (:name %) "RATING")- (get-in (nth stages 0) [:lib/stage-metadata :columns])))))- (testing "second stage (from Card 2)"- (is (=? {:name "RATING"- :display-name "Product → Rating"- :lib/original-join-alias "Product"}- (m/find-first #(= (:name %) "RATING")- (get-in (nth stages 1) [:lib/stage-metadata :columns])))))))(is (=? [{:display_name "Product → Rating"}{:display_name "Sum of Total"}](qp.preprocess/query->expected-cols query))))))diff --git a/test/metabase/query_processor/util/add_alias_info_test.clj b/test/metabase/query_processor/util/add_alias_info_test.cljindex 12991c88fed..deba39f61fc 100644--- a/test/metabase/query_processor/util/add_alias_info_test.clj+++ b/test/metabase/query_processor/util/add_alias_info_test.clj@@ -1116,8 +1116,12 @@:aggregation [[:sum $orders.quantity]]}:alias "Orders":condition [:= $id &Orders.orders.product-id]- ;; we can get title since product_id is remapped to title- :fields [&Orders.title+ ;; we can get title since product_id is remapped to title.+ ;;+ ;; TODO (Cam 9/16/25) -- says who? This is not something we support in+ ;; the FE.+ :fields [[:field %products.title {:join-alias "Orders"+ :source-field (meta/id :orders :product-id)}]&Orders.*sum/Integer]}]:fields [$title $category]:order-by [[:asc $id]]diff --git a/test/metabase/query_processor_test/field_ref_repro_test.clj b/test/metabase/query_processor_test/field_ref_repro_test.cljindex 4fdaae48e22..5f075d46056 100644--- a/test/metabase/query_processor_test/field_ref_repro_test.clj+++ b/test/metabase/query_processor_test/field_ref_repro_test.clj@@ -200,8 +200,8 @@;; The order of these columns seems to be 'flexible' (I would consider either to be correct), and;; I've seen both in two different branches of mine attempting to fix this bug. The order doesn't;; matter at all to the FE, so if this changes in the future it's ok. -- Cam- "j__PEOPLE__via__USER_ID__EMAIL" #_"j__EMAIL"- "PEOPLE__via__USER_ID__EMAIL"]+ "PEOPLE__via__USER_ID__EMAIL"+ "j__PEOPLE__via__USER_ID__EMAIL"](map :lib/desired-column-alias (mt/cols results))))(is (= [[1 ; <= orders.id1 ; <= orders.user-id@@ -221,8 +221,8 @@nil"2016-12-25T22:19:38.656Z"2- "[email protected]" ; (joined) orders.user-id --[remap]--> people.email (email of People row with ID = 1902)- "[email protected]"] ; orders.user-id --[remap]--> people.email (email of People row with ID = 1)+ "[email protected]" ; orders.user-id --[remap]--> people.email (email of People row with ID = 1)+ "[email protected]"] ; (joined) orders.user-id --[remap]--> people.email (email of People row with ID = 1902)[1114@@ -241,8 +241,8 @@nil"2017-02-04T10:16:00.936Z"1- "[email protected]"]]+ "[email protected]"]](mt/rows results))))))))))(deftest ^:parallel multi-stage-with-external-remapping-testdiff --git a/test/metabase/query_processor_test/remapping_test.clj b/test/metabase/query_processor_test/remapping_test.cljindex 518624d19de..9d41b9bd24d 100644--- a/test/metabase/query_processor_test/remapping_test.clj+++ b/test/metabase/query_processor_test/remapping_test.clj@@ -335,7 +335,7 @@:alias "Orders":condition [:= $id &Orders.orders.product_id];; we can get products.title since orders.product_id is remapped to title- :fields [&Orders.title+ :fields [[:field %products.title {:source-field (mt/id :orders :product_id), :join-alias "Orders"}]&Orders.*sum/Integer]}]:fields [$title $category]:order-by [[:asc $id]]@@ -466,25 +466,21 @@;;;; The order of remaps is not important to the FE. If it changes in the future that is ok.;;- ;; 2 remaps from the join against `VENUES`- "J__CATEGORIES__via__ID__NAME"- "J__CATEGORIES__via__CATEGORY_ID__NAME";; 2 remaps for the top-level query"CATEGORIES__via__CATEGORY_ID__NAME""CATEGORIES__via__ID__NAME"- ;; BROKEN! This is a duplicate and should not be returned. Interestingly enough, both Lib and QP- ;; incorrectly calculate the set of returned columns and both include this. (Probably because Lib and QP- ;; use mostly the same code these days.)- "J__CATEGORIES__via__ID__NAME_2"]+ ;; 2 remaps from the join against `VENUES`+ "J__CATEGORIES__via__ID__NAME"+ "J__CATEGORIES__via__CATEGORY_ID__NAME"](map :lib/desired-column-alias (mt/cols results))));; The extra incorrect duplicate column seems to be sorta indetermiate? I've seen it match the value of;; `J__CATEGORIES__via__ID__NAME` and `J__CATEGORIES__via__CATEGORY_ID__NAME` in different test runs and I'm not;; sure why. Not bothering to debug since it's not even supposed to be returned anyway.;;- ;; <top-level :fields> <join> <join remaps> <fields remaps> <incorrect duplicate>- (is (=? [[11 2 "Stout Burgers & Beers" 1 "Red Medicine" 4 10.0646 -165.374 3 "African" "Asian" "Burger" "American" string?]- [11 3 "The Apple Pan" 2 "Stout Burgers & Beers" 11 34.0996 -118.329 2 "American" "Burger" "Burger" "Artisan" string?]- [29 4 "Wurstküche" 3 "The Apple Pan" 11 34.0406 -118.428 2 "Artisan" "Burger" "German" "Asian" string?]]+ ;; <top-level :fields> <join> <fields remaps> <join remaps>+ (is (=? [[11 2 "Stout Burgers & Beers" 1 "Red Medicine" 4 10.0646 -165.374 3 "Burger" "American" "African" "Asian"]+ [11 3 "The Apple Pan" 2 "Stout Burgers & Beers" 11 34.0996 -118.329 2 "Burger" "Artisan" "American" "Burger"]+ [29 4 "Wurstküche" 3 "The Apple Pan" 11 34.0406 -118.428 2 "German" "Asian" "Artisan" "Burger"]](mt/rows results))))))(deftest ^:parallel explicit-join-with-fields-and-implicitly-joined-remaps-test